All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
@ 2019-06-19 22:50 Kim, Jonathan
       [not found] ` <20190619224943.130022-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  0 siblings, 1 reply; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-19 22:50 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org; +Cc: Kim, Jonathan

v4: fixed kzalloc error check and modified df func init to return error code

v3: fixed cleanup by adding fini to free up adev df config counters

v2: simplified by removing xgmi references in function names and moving to
generic df function names.  fixed issue by removing hardcoded cake tx data
events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
error code.

add pmu attribute groups and structures for perf events.
add sysfs to track available df perfmon counters
fix overflow handling in perfmon counter reads.

Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  19 +-
 drivers/gpu/drm/amd/amdgpu/df_v1_7.c |   3 +-
 drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 466 +++++++++++++--------------
 drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c   |   8 +
 5 files changed, 254 insertions(+), 261 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d355e9a09ad1..5780e157fd72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -681,7 +681,8 @@ struct amdgpu_nbio_funcs {
 };
 
 struct amdgpu_df_funcs {
-	void (*init)(struct amdgpu_device *adev);
+	int (*init)(struct amdgpu_device *adev);
+	void (*fini)(struct amdgpu_device *adev);
 	void (*enable_broadcast_mode)(struct amdgpu_device *adev,
 				      bool enable);
 	u32 (*get_fb_channel_number)(struct amdgpu_device *adev);
@@ -962,6 +963,7 @@ struct amdgpu_device {
 	long				compute_timeout;
 
 	uint64_t			unique_id;
+	uint64_t			*df_perfmon_config_assign_mask;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
 #endif
 
 #include "amdgpu_object.h"
+
+/* used by df_v3_6.c and amdgpu_pmu.c */
+#define AMDGPU_PMU_ATTR(_name, _object)				\
+static ssize_t								\
+_name##_show(struct device *dev,					\
+			       struct device_attribute *attr,		\
+			       char *page)				\
+{									\
+	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
+	return sprintf(page, _object "\n");				\
+}									\
+									\
+static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
+
 #endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
index 9935371db7ce..cc964d52dee4 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
@@ -29,8 +29,9 @@
 
 static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
 
-static void df_v1_7_init (struct amdgpu_device *adev)
+static int df_v1_7_init (struct amdgpu_device *adev)
 {
+	return 0;
 }
 
 static void df_v1_7_enable_broadcast_mode(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index 8c09bf994acd..cae66bc59986 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -30,8 +30,118 @@
 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
 				       16, 32, 0, 0, 0, 2, 4, 8};
 
-static void df_v3_6_init(struct amdgpu_device *adev)
+/* init df format attrs */
+AMDGPU_PMU_ATTR(event,		"config:0-7");
+AMDGPU_PMU_ATTR(instance,	"config:8-15");
+AMDGPU_PMU_ATTR(umask,		"config:16-23");
+
+/* df format attributes  */
+static struct attribute *df_v3_6_format_attrs[] = {
+	&pmu_attr_event.attr,
+	&pmu_attr_instance.attr,
+	&pmu_attr_umask.attr,
+	NULL
+};
+
+/* df format attribute group */
+static struct attribute_group df_v3_6_format_attr_group = {
+	.name = "format",
+	.attrs = df_v3_6_format_attrs,
+};
+
+/* df event attrs */
+AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
+		      "event=0x7,instance=0x46,umask=0x2");
+AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
+		      "event=0x7,instance=0x47,umask=0x2");
+AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
+		      "event=0x7,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
+		      "event=0x7,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x46,umask=0x8");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x47,umask=0x8");
+
+/* df event attributes  */
+static struct attribute *df_v3_6_event_attrs[] = {
+	&pmu_attr_cake0_pcsout_txdata.attr,
+	&pmu_attr_cake1_pcsout_txdata.attr,
+	&pmu_attr_cake0_pcsout_txmeta.attr,
+	&pmu_attr_cake1_pcsout_txmeta.attr,
+	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
+	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
+	NULL
+};
+
+/* df event attribute group */
+static struct attribute_group df_v3_6_event_attr_group = {
+	.name = "events",
+	.attrs = df_v3_6_event_attrs
+};
+
+/* df event attr groups  */
+const struct attribute_group *df_v3_6_attr_groups[] = {
+		&df_v3_6_format_attr_group,
+		&df_v3_6_event_attr_group,
+		NULL
+};
+
+/* get the number of df counters available */
+static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *ddev;
+	int i, count;
+
+	ddev = dev_get_drvdata(dev);
+	adev = ddev->dev_private;
+	count = 0;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0)
+			count++;
+	}
+
+	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
+}
+
+/* device attr for available perfmon counters */
+static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
+
+/* init perfmons */
+static int df_v3_6_init(struct amdgpu_device *adev)
+{
+	int i, ret;
+
+	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
+	if (ret)
+		DRM_ERROR("failed to create file for available df counters\n");
+
+	adev->df_perfmon_config_assign_mask =
+		kzalloc(sizeof(uint64_t) * DF_V3_6_MAX_COUNTERS, GFP_KERNEL);
+
+	if (!adev->df_perfmon_config_assign_mask)
+		return -ENOMEM;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++)
+		adev->df_perfmon_config_assign_mask[i] = 0;
+
+	return 0;
+}
+
+static void df_v3_6_fini(struct amdgpu_device *adev)
 {
+	kfree(adev->df_perfmon_config_assign_mask);
+	adev->df_perfmon_config_assign_mask = NULL;
 }
 
 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
@@ -105,28 +215,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 		*flags |= AMD_CG_SUPPORT_DF_MGCG;
 }
 
-/* hold counter assignment per gpu struct */
-struct df_v3_6_event_mask {
-		struct amdgpu_device gpu;
-		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
-};
-
 /* get assigned df perfmon ctr as int */
-static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
-				      uint64_t config,
-				      int *counter)
+static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
-	struct df_v3_6_event_mask *mask;
 	int i;
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
-			*counter = i;
-			return;
-		}
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if ((config & 0x0FFFFFFUL) ==
+					adev->df_perfmon_config_assign_mask[i])
+			return i;
 	}
+
+	return -EINVAL;
 }
 
 /* get address based on counter assignment */
@@ -136,10 +237,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 				 uint32_t *lo_base_addr,
 				 uint32_t *hi_base_addr)
 {
-
-	int target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr < 0)
 		return;
@@ -177,40 +275,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
 }
 
 /* get control counter settings i.e. address and values to set */
-static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 					  uint64_t config,
 					  uint32_t *lo_base_addr,
 					  uint32_t *hi_base_addr,
 					  uint32_t *lo_val,
 					  uint32_t *hi_val)
 {
-
-	uint32_t eventsel, instance, unitmask;
-	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
-
 	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 
-	if (lo_val == NULL || hi_val == NULL)
-		return;
-
 	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
-		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
+		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 				*lo_base_addr, *hi_base_addr);
-		return;
+		return -ENXIO;
 	}
 
-	eventsel = GET_EVENT(config);
-	instance = GET_INSTANCE(config);
-	unitmask = GET_UNITMASK(config);
-
-	es_5_0 = eventsel & 0x3FUL;
-	es_13_6 = instance;
-	es_13_0 = (es_13_6 << 6) + es_5_0;
-	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
-	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
-	es_7_0 = es_13_0 & 0x0FFUL;
-	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
-	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
+	if (lo_val && hi_val) {
+		uint32_t eventsel, instance, unitmask;
+		uint32_t instance_10, instance_5432, instance_76;
+
+		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
+		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
+		instance = DF_V3_6_GET_INSTANCE(config);
+
+		instance_10 = instance & 0x3;
+		instance_5432 = (instance >> 2) & 0xf;
+		instance_76 = (instance >> 6) & 0x3;
+
+		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
+		*hi_val = (instance_76 << 29) | instance_5432;
+	}
+
+	return 0;
 }
 
 /* assign df performance counters for read */
@@ -218,26 +314,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 				   uint64_t config,
 				   int *is_assigned)
 {
-
-	struct df_v3_6_event_mask *mask;
 	int i, target_cntr;
 
-	target_cntr = -1;
-
 	*is_assigned = 0;
 
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0) {
 		*is_assigned = 1;
 		return 0;
 	}
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if (mask->config_assign_mask[i] == 0ULL) {
-			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
+			adev->df_perfmon_config_assign_mask[i] =
+							config & 0x0FFFFFFUL;
 			return 0;
 		}
 	}
@@ -249,66 +340,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 				     uint64_t config)
 {
-
-	struct df_v3_6_event_mask *mask;
-	int target_cntr;
-
-	target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
-
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0)
-		mask->config_assign_mask[target_cntr] = 0ULL;
-
+		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 }
 
-/*
- * get xgmi link counters via programmable data fabric (df) counters (max 4)
- * using cake tx event.
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @count -> counters to pass
- *
- */
 
-static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       uint64_t *count)
-{
-	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
-				      &hi_base_addr);
-
-	if ((lo_base_addr == 0) || (hi_base_addr == 0))
-		return;
-
-	lo_val = RREG32_PCIE(lo_base_addr);
-	hi_val = RREG32_PCIE(hi_base_addr);
-
-	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
-}
-
-/*
- * reset xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
-static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
-					 int instance)
+static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
+					 uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr;
-	uint64_t config;
-
-	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
 
 	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 				      &hi_base_addr);
@@ -320,185 +362,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
 	WREG32_PCIE(hi_base_addr, 0UL);
 }
 
-/*
- * add xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
 
-static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
-				      int instance)
+static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
 	int ret, is_assigned;
 
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
 	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
 
 	if (ret || is_assigned)
 		return ret;
 
-	df_v3_6_pmc_get_ctrl_settings(adev,
+	ret = df_v3_6_pmc_get_ctrl_settings(adev,
 			config,
 			&lo_base_addr,
 			&hi_base_addr,
 			&lo_val,
 			&hi_val);
 
+	if (ret)
+		return ret;
+
+	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
 	WREG32_PCIE(lo_base_addr, lo_val);
 	WREG32_PCIE(hi_base_addr, hi_val);
 
 	return ret;
 }
 
-
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either resume or assign event via df perfmon
- *
- */
-
-static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
-					int instance,
-					int is_enable)
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+			     int is_enable)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-	int ret;
-
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
+	int ret = 0;
 
-	if (is_enable) {
-
-		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
-
-		if (ret)
-			return ret;
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
 
-	} else {
+		df_v3_6_reset_perfmon_cntr(adev, config);
 
-		config = GET_INSTANCE_CONFIG(instance);
+		if (is_enable) {
+			ret = df_v3_6_add_perfmon_cntr(adev, config);
+		} else {
+			ret = df_v3_6_pmc_get_ctrl_settings(adev,
+					config,
+					&lo_base_addr,
+					&hi_base_addr,
+					NULL,
+					NULL);
 
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
+			if (ret)
+				return ret;
 
-		if (lo_base_addr == 0)
-			return -EINVAL;
+			lo_val = RREG32_PCIE(lo_base_addr);
 
-		lo_val = RREG32_PCIE(lo_base_addr);
+			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+		}
 
-		ret = 0;
+		break;
+	default:
+		break;
 	}
 
 	return ret;
-
 }
 
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either pause or unassign event via df perfmon
- *
- */
-
-static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       int is_disable)
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+			    int is_disable)
 {
-
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	if (is_disable) {
-		df_v3_6_reset_xgmi_link_cntr(adev, instance);
-		df_v3_6_pmc_release_cntr(adev, config);
-	} else {
-
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
-
-		if ((lo_base_addr == 0) || (hi_base_addr == 0))
-			return -EINVAL;
-
-		lo_val = RREG32_PCIE(lo_base_addr);
-
-		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
-	}
-
-	return 0;
-}
-
-static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
-			     int is_enable)
-{
-	int xgmi_tx_link, ret = 0;
+	int ret = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-		if (xgmi_tx_link >= 0)
-			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
-						      is_enable);
+		ret = df_v3_6_pmc_get_ctrl_settings(adev,
+			config,
+			&lo_base_addr,
+			&hi_base_addr,
+			NULL,
+			NULL);
 
 		if (ret)
 			return ret;
 
-		ret = 0;
-		break;
-	default:
-		break;
-	}
+		lo_val = RREG32_PCIE(lo_base_addr);
 
-	return ret;
-}
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
-			    int is_disable)
-{
-	int xgmi_tx_link, ret = 0;
+		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
 
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-			if (xgmi_tx_link >= 0) {
-				ret = df_v3_6_stop_xgmi_link_cntr(adev,
-								  xgmi_tx_link,
-								  is_disable);
-				if (ret)
-					return ret;
-			}
-
-			ret = 0;
-			break;
+		if (is_disable)
+			df_v3_6_pmc_release_cntr(adev, config);
+
+		break;
 	default:
 		break;
 	}
@@ -510,28 +473,39 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 				  uint64_t config,
 				  uint64_t *count)
 {
-
-	int xgmi_tx_link;
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	*count = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 
-		if (xgmi_tx_link >= 0) {
-			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
-			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
-		}
+		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+		if ((lo_base_addr == 0) || (hi_base_addr == 0))
+			return;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+		hi_val = RREG32_PCIE(hi_base_addr);
+
+		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+
+		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
+			*count = 0;
+
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 
 		break;
+
 	default:
 		break;
 	}
-
 }
 
 const struct amdgpu_df_funcs df_v3_6_funcs = {
 	.init = df_v3_6_init,
+	.fini = df_v3_6_fini,
 	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
 	.get_fb_channel_number = df_v3_6_get_fb_channel_number,
 	.get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
index fcffd807764d..76998541bc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
 };
 
 /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
-#define AMDGPU_DF_MAX_COUNTERS		4
+#define DF_V3_6_MAX_COUNTERS		4
 
 /* get flags from df perfmon config */
-#define GET_EVENT(x)			(x & 0xFFUL)
-#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
-#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
-#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
-					| ((0x046ULL + x) << 8) \
-					| (0x02 << 16))
-
-/* df event conf macros */
-#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
-#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
+#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
+#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
+#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
+#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
 
+extern const struct attribute_group *df_v3_6_attr_groups[];
 extern const struct amdgpu_df_funcs df_v3_6_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 78bd4fc07bab..551c77749b2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
 static int soc15_common_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
 
 	/* enable pcie gen2/3 link */
 	soc15_pcie_gen3_enable(adev);
@@ -1079,6 +1080,11 @@ static int soc15_common_hw_init(void *handle)
 	 */
 	if (adev->nbio_funcs->remap_hdp_registers)
 		adev->nbio_funcs->remap_hdp_registers(adev);
+
+	ret = adev->df_funcs->init(adev);
+	if (ret)
+		return ret;
+
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 	/* HW doorbell routing policy: doorbell writing not
@@ -1100,6 +1106,8 @@ static int soc15_common_hw_fini(void *handle)
 	if (amdgpu_sriov_vf(adev))
 		xgpu_ai_mailbox_put_irq(adev);
 
+	adev->df_funcs->fini(adev);
+
 	return 0;
 }
 
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* [PATCH 2/2] drm/amdgpu: add pmu counters
       [not found] ` <20190619224943.130022-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-19 22:50   ` Kim, Jonathan
       [not found]     ` <20190619224943.130022-2-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-19 23:00   ` [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons Kim, Jonathan
  2019-06-19 23:27   ` Kuehling, Felix
  2 siblings, 1 reply; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-19 22:50 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org; +Cc: Kim, Jonathan

adding perf event counters

Change-Id: Ib8800c2d652e3e788872bd721458ac5422859b62
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile        |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c    | 280 +++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h    |  37 +++
 5 files changed, 325 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 57ce44cc3226..4c9fd2645f64 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -54,7 +54,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
 	amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \
 	amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
-	amdgpu_vm_sdma.o
+	amdgpu_vm_sdma.o amdgpu_pmu.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 5780e157fd72..90a159079912 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -964,6 +964,7 @@ struct amdgpu_device {
 
 	uint64_t			unique_id;
 	uint64_t			*df_perfmon_config_assign_mask;
+	struct mutex		df_perfmon_lock;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 003350a2d299..bcfabd025f5d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -61,6 +61,7 @@
 
 #include "amdgpu_xgmi.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_pmu.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
@@ -2757,6 +2758,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		return r;
 	}
 
+	r = amdgpu_pmu_init(adev);
+	if (r)
+		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
+
 	return 0;
 
 failed:
@@ -2822,6 +2827,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	amdgpu_debugfs_regs_cleanup(adev);
 	device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
 	amdgpu_ucode_sysfs_fini(adev);
+	amdgpu_pmu_fini(adev);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
new file mode 100644
index 000000000000..73fd70a59c8b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#include <linux/perf_event.h>
+#include <linux/init.h>
+#include "amdgpu.h"
+#include "amdgpu_pmu.h"
+#include "df_v3_6.h"
+
+#define PMU_NAME_SIZE 32
+
+/* record to keep track of pmu entry per pmu type per device */
+struct amdgpu_pmu_entry {
+	struct list_head entry;
+	struct amdgpu_device *adev;
+	struct pmu pmu;
+	unsigned int pmu_perf_type;
+};
+
+static LIST_HEAD(amdgpu_pmu_list);
+
+
+/* initialize perf counter */
+static int amdgpu_perf_event_init(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	/* test the event attr type check for PMU enumeration */
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	/* update the hw_perf_event struct with config data */
+	hwc->conf = event->attr.config;
+
+	return 0;
+}
+
+/* start perf counter */
+static void amdgpu_perf_start(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+		return;
+
+	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+	hwc->state = 0;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		if (!(flags & PERF_EF_RELOAD))
+			pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+
+		pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 0);
+		break;
+	default:
+		break;
+	}
+
+	perf_event_update_userpage(event);
+
+}
+
+/* read perf counter */
+static void amdgpu_perf_read(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	u64 count, prev;
+
+	do {
+		prev = local64_read(&hwc->prev_count);
+
+		switch (pe->pmu_perf_type) {
+		case PERF_TYPE_AMDGPU_DF:
+			pe->adev->df_funcs->pmc_get_count(pe->adev, hwc->conf,
+							  &count);
+			break;
+		default:
+			count = 0;
+			break;
+		};
+	} while (local64_cmpxchg(&hwc->prev_count, prev, count) != prev);
+
+	local64_add(count - prev, &event->count);
+}
+
+/* stop perf counter */
+static void amdgpu_perf_stop(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	if (hwc->state & PERF_HES_UPTODATE)
+		return;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 0);
+		break;
+	default:
+		break;
+	};
+
+	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+	hwc->state |= PERF_HES_STOPPED;
+
+	if (hwc->state & PERF_HES_UPTODATE)
+		return;
+
+	amdgpu_perf_read(event);
+	hwc->state |= PERF_HES_UPTODATE;
+}
+
+/* add perf counter  */
+static int amdgpu_perf_add(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int retval;
+
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		retval = pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+		break;
+	default:
+		return 0;
+	};
+
+	if (retval)
+		return retval;
+
+	if (flags & PERF_EF_START)
+		amdgpu_perf_start(event, PERF_EF_RELOAD);
+
+	return retval;
+
+}
+
+/* delete perf counter  */
+static void amdgpu_perf_del(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	amdgpu_perf_stop(event, PERF_EF_UPDATE);
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 1);
+		break;
+	default:
+		break;
+	};
+
+	perf_event_update_userpage(event);
+}
+
+/* vega20 pmus */
+
+/* init pmu tracking per pmu type */
+int init_pmu_by_type(struct amdgpu_device *adev,
+		  const struct attribute_group *attr_groups[],
+		  char *pmu_type_name, char *pmu_file_prefix,
+		  unsigned int pmu_perf_type)
+{
+	char pmu_name[PMU_NAME_SIZE];
+	struct amdgpu_pmu_entry *pmu_entry;
+	int ret = 0;
+
+	pmu_entry = kzalloc(sizeof(struct amdgpu_pmu_entry), GFP_KERNEL);
+
+	if (!pmu_entry)
+		return -ENOMEM;
+
+	pmu_entry->adev = adev;
+	pmu_entry->pmu = (struct pmu){
+		.event_init = amdgpu_perf_event_init,
+		.add = amdgpu_perf_add,
+		.del = amdgpu_perf_del,
+		.start = amdgpu_perf_start,
+		.stop = amdgpu_perf_stop,
+		.read = amdgpu_perf_read,
+		.task_ctx_nr = perf_invalid_context,
+	};
+
+	pmu_entry->pmu.attr_groups = attr_groups;
+	pmu_entry->pmu_perf_type = pmu_perf_type;
+	snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d",
+				pmu_file_prefix, adev->ddev->primary->index);
+
+	ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1);
+
+	if (ret) {
+		kfree(pmu_entry);
+		pr_warn("Error initializing AMDGPU %s PMUs.\n", pmu_type_name);
+		return ret;
+	}
+
+	pr_info("Detected AMDGPU %s Counters. # of Counters = %d.\n",
+			pmu_type_name, DF_V3_6_MAX_COUNTERS);
+
+	list_add_tail(&pmu_entry->entry, &amdgpu_pmu_list);
+
+	return 0;
+}
+
+/* init amdgpu_pmu */
+int amdgpu_pmu_init(struct amdgpu_device *adev)
+{
+
+	int ret = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		/* init df */
+		ret = init_pmu_by_type(adev, df_v3_6_attr_groups,
+				"DF", "amdgpu_df", PERF_TYPE_AMDGPU_DF);
+
+		/* other pmu types go here*/
+		break;
+	default:
+		return 0;
+	}
+
+	return 0;
+
+}
+
+
+/* destroy all pmu data associated with target device */
+void amdgpu_pmu_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_pmu_entry *pe, *temp;
+
+	list_for_each_entry_safe(pe, temp, &amdgpu_pmu_list, entry) {
+		if (pe->adev == adev) {
+			list_del(&pe->entry);
+			perf_pmu_unregister(&temp->pmu);
+			kfree(temp);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
new file mode 100644
index 000000000000..7dddb7160a11
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#ifndef _AMDGPU_PMU_H_
+#define _AMDGPU_PMU_H_
+
+enum amdgpu_pmu_perf_type {
+	PERF_TYPE_AMDGPU_DF = 0,
+	PERF_TYPE_AMDGPU_MAX
+};
+
+int amdgpu_pmu_init(struct amdgpu_device *adev);
+void amdgpu_pmu_fini(struct amdgpu_device *adev);
+
+#endif /* _AMDGPU_PMU_H_ */
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* RE: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found] ` <20190619224943.130022-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-19 22:50   ` [PATCH 2/2] drm/amdgpu: add pmu counters Kim, Jonathan
@ 2019-06-19 23:00   ` Kim, Jonathan
  2019-06-19 23:27   ` Kuehling, Felix
  2 siblings, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-19 23:00 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org,
	Kuehling, Felix

Hi Felix - Patch 1 as promised for review.

Jon

-----Original Message-----
From: Kim, Jonathan <Jonathan.Kim@amd.com> 
Sent: Wednesday, June 19, 2019 6:50 PM
To: amd-gfx@lists.freedesktop.org
Cc: Kim, Jonathan <Jonathan.Kim@amd.com>; Kim, Jonathan <Jonathan.Kim@amd.com>
Subject: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons

v4: fixed kzalloc error check and modified df func init to return error code

v3: fixed cleanup by adding fini to free up adev df config counters

v2: simplified by removing xgmi references in function names and moving to generic df function names.  fixed issue by removing hardcoded cake tx data events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return error code.

add pmu attribute groups and structures for perf events.
add sysfs to track available df perfmon counters fix overflow handling in perfmon counter reads.

Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  19 +-
 drivers/gpu/drm/amd/amdgpu/df_v1_7.c |   3 +-
 drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 466 +++++++++++++--------------  drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c   |   8 +
 5 files changed, 254 insertions(+), 261 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d355e9a09ad1..5780e157fd72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -681,7 +681,8 @@ struct amdgpu_nbio_funcs {  };
 
 struct amdgpu_df_funcs {
-	void (*init)(struct amdgpu_device *adev);
+	int (*init)(struct amdgpu_device *adev);
+	void (*fini)(struct amdgpu_device *adev);
 	void (*enable_broadcast_mode)(struct amdgpu_device *adev,
 				      bool enable);
 	u32 (*get_fb_channel_number)(struct amdgpu_device *adev); @@ -962,6 +963,7 @@ struct amdgpu_device {
 	long				compute_timeout;
 
 	uint64_t			unique_id;
+	uint64_t			*df_perfmon_config_assign_mask;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return  #endif
 
 #include "amdgpu_object.h"
+
+/* used by df_v3_6.c and amdgpu_pmu.c */
+#define AMDGPU_PMU_ATTR(_name, _object)				\
+static ssize_t								\
+_name##_show(struct device *dev,					\
+			       struct device_attribute *attr,		\
+			       char *page)				\
+{									\
+	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
+	return sprintf(page, _object "\n");				\
+}									\
+									\
+static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
+
 #endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
index 9935371db7ce..cc964d52dee4 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
@@ -29,8 +29,9 @@
 
 static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
 
-static void df_v1_7_init (struct amdgpu_device *adev)
+static int df_v1_7_init (struct amdgpu_device *adev)
 {
+	return 0;
 }
 
 static void df_v1_7_enable_broadcast_mode(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index 8c09bf994acd..cae66bc59986 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -30,8 +30,118 @@
 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
 				       16, 32, 0, 0, 0, 2, 4, 8};
 
-static void df_v3_6_init(struct amdgpu_device *adev)
+/* init df format attrs */
+AMDGPU_PMU_ATTR(event,		"config:0-7");
+AMDGPU_PMU_ATTR(instance,	"config:8-15");
+AMDGPU_PMU_ATTR(umask,		"config:16-23");
+
+/* df format attributes  */
+static struct attribute *df_v3_6_format_attrs[] = {
+	&pmu_attr_event.attr,
+	&pmu_attr_instance.attr,
+	&pmu_attr_umask.attr,
+	NULL
+};
+
+/* df format attribute group */
+static struct attribute_group df_v3_6_format_attr_group = {
+	.name = "format",
+	.attrs = df_v3_6_format_attrs,
+};
+
+/* df event attrs */
+AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
+		      "event=0x7,instance=0x46,umask=0x2");
+AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
+		      "event=0x7,instance=0x47,umask=0x2");
+AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
+		      "event=0x7,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
+		      "event=0x7,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x46,umask=0x8");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x47,umask=0x8");
+
+/* df event attributes  */
+static struct attribute *df_v3_6_event_attrs[] = {
+	&pmu_attr_cake0_pcsout_txdata.attr,
+	&pmu_attr_cake1_pcsout_txdata.attr,
+	&pmu_attr_cake0_pcsout_txmeta.attr,
+	&pmu_attr_cake1_pcsout_txmeta.attr,
+	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
+	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
+	NULL
+};
+
+/* df event attribute group */
+static struct attribute_group df_v3_6_event_attr_group = {
+	.name = "events",
+	.attrs = df_v3_6_event_attrs
+};
+
+/* df event attr groups  */
+const struct attribute_group *df_v3_6_attr_groups[] = {
+		&df_v3_6_format_attr_group,
+		&df_v3_6_event_attr_group,
+		NULL
+};
+
+/* get the number of df counters available */ static ssize_t 
+df_v3_6_get_df_cntr_avail(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *ddev;
+	int i, count;
+
+	ddev = dev_get_drvdata(dev);
+	adev = ddev->dev_private;
+	count = 0;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0)
+			count++;
+	}
+
+	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
+}
+
+/* device attr for available perfmon counters */ static 
+DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
+
+/* init perfmons */
+static int df_v3_6_init(struct amdgpu_device *adev) {
+	int i, ret;
+
+	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
+	if (ret)
+		DRM_ERROR("failed to create file for available df counters\n");
+
+	adev->df_perfmon_config_assign_mask =
+		kzalloc(sizeof(uint64_t) * DF_V3_6_MAX_COUNTERS, GFP_KERNEL);
+
+	if (!adev->df_perfmon_config_assign_mask)
+		return -ENOMEM;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++)
+		adev->df_perfmon_config_assign_mask[i] = 0;
+
+	return 0;
+}
+
+static void df_v3_6_fini(struct amdgpu_device *adev)
 {
+	kfree(adev->df_perfmon_config_assign_mask);
+	adev->df_perfmon_config_assign_mask = NULL;
 }
 
 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev, @@ -105,28 +215,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 		*flags |= AMD_CG_SUPPORT_DF_MGCG;
 }
 
-/* hold counter assignment per gpu struct */ -struct df_v3_6_event_mask {
-		struct amdgpu_device gpu;
-		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
-};
-
 /* get assigned df perfmon ctr as int */ -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
-				      uint64_t config,
-				      int *counter)
+static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
-	struct df_v3_6_event_mask *mask;
 	int i;
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
-			*counter = i;
-			return;
-		}
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if ((config & 0x0FFFFFFUL) ==
+					adev->df_perfmon_config_assign_mask[i])
+			return i;
 	}
+
+	return -EINVAL;
 }
 
 /* get address based on counter assignment */ @@ -136,10 +237,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 				 uint32_t *lo_base_addr,
 				 uint32_t *hi_base_addr)
 {
-
-	int target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr < 0)
 		return;
@@ -177,40 +275,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,  }
 
 /* get control counter settings i.e. address and values to set */ -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 					  uint64_t config,
 					  uint32_t *lo_base_addr,
 					  uint32_t *hi_base_addr,
 					  uint32_t *lo_val,
 					  uint32_t *hi_val)
 {
-
-	uint32_t eventsel, instance, unitmask;
-	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
-
 	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 
-	if (lo_val == NULL || hi_val == NULL)
-		return;
-
 	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
-		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
+		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 				*lo_base_addr, *hi_base_addr);
-		return;
+		return -ENXIO;
 	}
 
-	eventsel = GET_EVENT(config);
-	instance = GET_INSTANCE(config);
-	unitmask = GET_UNITMASK(config);
-
-	es_5_0 = eventsel & 0x3FUL;
-	es_13_6 = instance;
-	es_13_0 = (es_13_6 << 6) + es_5_0;
-	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
-	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
-	es_7_0 = es_13_0 & 0x0FFUL;
-	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
-	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
+	if (lo_val && hi_val) {
+		uint32_t eventsel, instance, unitmask;
+		uint32_t instance_10, instance_5432, instance_76;
+
+		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
+		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
+		instance = DF_V3_6_GET_INSTANCE(config);
+
+		instance_10 = instance & 0x3;
+		instance_5432 = (instance >> 2) & 0xf;
+		instance_76 = (instance >> 6) & 0x3;
+
+		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
+		*hi_val = (instance_76 << 29) | instance_5432;
+	}
+
+	return 0;
 }
 
 /* assign df performance counters for read */ @@ -218,26 +314,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 				   uint64_t config,
 				   int *is_assigned)
 {
-
-	struct df_v3_6_event_mask *mask;
 	int i, target_cntr;
 
-	target_cntr = -1;
-
 	*is_assigned = 0;
 
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0) {
 		*is_assigned = 1;
 		return 0;
 	}
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if (mask->config_assign_mask[i] == 0ULL) {
-			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
+			adev->df_perfmon_config_assign_mask[i] =
+							config & 0x0FFFFFFUL;
 			return 0;
 		}
 	}
@@ -249,66 +340,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,  static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 				     uint64_t config)
 {
-
-	struct df_v3_6_event_mask *mask;
-	int target_cntr;
-
-	target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
-
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0)
-		mask->config_assign_mask[target_cntr] = 0ULL;
-
+		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 }
 
-/*
- * get xgmi link counters via programmable data fabric (df) counters (max 4)
- * using cake tx event.
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @count -> counters to pass
- *
- */
 
-static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       uint64_t *count)
-{
-	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
-				      &hi_base_addr);
-
-	if ((lo_base_addr == 0) || (hi_base_addr == 0))
-		return;
-
-	lo_val = RREG32_PCIE(lo_base_addr);
-	hi_val = RREG32_PCIE(hi_base_addr);
-
-	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
-}
-
-/*
- * reset xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
-static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
-					 int instance)
+static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
+					 uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr;
-	uint64_t config;
-
-	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
 
 	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 				      &hi_base_addr);
@@ -320,185 +362,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
 	WREG32_PCIE(hi_base_addr, 0UL);
 }
 
-/*
- * add xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
 
-static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
-				      int instance)
+static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
 	int ret, is_assigned;
 
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
 	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
 
 	if (ret || is_assigned)
 		return ret;
 
-	df_v3_6_pmc_get_ctrl_settings(adev,
+	ret = df_v3_6_pmc_get_ctrl_settings(adev,
 			config,
 			&lo_base_addr,
 			&hi_base_addr,
 			&lo_val,
 			&hi_val);
 
+	if (ret)
+		return ret;
+
+	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
 	WREG32_PCIE(lo_base_addr, lo_val);
 	WREG32_PCIE(hi_base_addr, hi_val);
 
 	return ret;
 }
 
-
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either resume or assign event via df perfmon
- *
- */
-
-static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
-					int instance,
-					int is_enable)
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+			     int is_enable)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-	int ret;
-
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
+	int ret = 0;
 
-	if (is_enable) {
-
-		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
-
-		if (ret)
-			return ret;
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
 
-	} else {
+		df_v3_6_reset_perfmon_cntr(adev, config);
 
-		config = GET_INSTANCE_CONFIG(instance);
+		if (is_enable) {
+			ret = df_v3_6_add_perfmon_cntr(adev, config);
+		} else {
+			ret = df_v3_6_pmc_get_ctrl_settings(adev,
+					config,
+					&lo_base_addr,
+					&hi_base_addr,
+					NULL,
+					NULL);
 
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
+			if (ret)
+				return ret;
 
-		if (lo_base_addr == 0)
-			return -EINVAL;
+			lo_val = RREG32_PCIE(lo_base_addr);
 
-		lo_val = RREG32_PCIE(lo_base_addr);
+			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+		}
 
-		ret = 0;
+		break;
+	default:
+		break;
 	}
 
 	return ret;
-
 }
 
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either pause or unassign event via df perfmon
- *
- */
-
-static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       int is_disable)
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+			    int is_disable)
 {
-
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	if (is_disable) {
-		df_v3_6_reset_xgmi_link_cntr(adev, instance);
-		df_v3_6_pmc_release_cntr(adev, config);
-	} else {
-
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
-
-		if ((lo_base_addr == 0) || (hi_base_addr == 0))
-			return -EINVAL;
-
-		lo_val = RREG32_PCIE(lo_base_addr);
-
-		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
-	}
-
-	return 0;
-}
-
-static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
-			     int is_enable)
-{
-	int xgmi_tx_link, ret = 0;
+	int ret = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-		if (xgmi_tx_link >= 0)
-			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
-						      is_enable);
+		ret = df_v3_6_pmc_get_ctrl_settings(adev,
+			config,
+			&lo_base_addr,
+			&hi_base_addr,
+			NULL,
+			NULL);
 
 		if (ret)
 			return ret;
 
-		ret = 0;
-		break;
-	default:
-		break;
-	}
+		lo_val = RREG32_PCIE(lo_base_addr);
 
-	return ret;
-}
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
-			    int is_disable)
-{
-	int xgmi_tx_link, ret = 0;
+		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
 
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-			if (xgmi_tx_link >= 0) {
-				ret = df_v3_6_stop_xgmi_link_cntr(adev,
-								  xgmi_tx_link,
-								  is_disable);
-				if (ret)
-					return ret;
-			}
-
-			ret = 0;
-			break;
+		if (is_disable)
+			df_v3_6_pmc_release_cntr(adev, config);
+
+		break;
 	default:
 		break;
 	}
@@ -510,28 +473,39 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 				  uint64_t config,
 				  uint64_t *count)
 {
-
-	int xgmi_tx_link;
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	*count = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 
-		if (xgmi_tx_link >= 0) {
-			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
-			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
-		}
+		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+		if ((lo_base_addr == 0) || (hi_base_addr == 0))
+			return;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+		hi_val = RREG32_PCIE(hi_base_addr);
+
+		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+
+		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
+			*count = 0;
+
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 
 		break;
+
 	default:
 		break;
 	}
-
 }
 
 const struct amdgpu_df_funcs df_v3_6_funcs = {
 	.init = df_v3_6_init,
+	.fini = df_v3_6_fini,
 	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
 	.get_fb_channel_number = df_v3_6_get_fb_channel_number,
 	.get_hbm_channel_number = df_v3_6_get_hbm_channel_number, diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
index fcffd807764d..76998541bc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
 };
 
 /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
-#define AMDGPU_DF_MAX_COUNTERS		4
+#define DF_V3_6_MAX_COUNTERS		4
 
 /* get flags from df perfmon config */
-#define GET_EVENT(x)			(x & 0xFFUL)
-#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
-#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
-#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
-					| ((0x046ULL + x) << 8) \
-					| (0x02 << 16))
-
-/* df event conf macros */
-#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
-#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
+#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
+#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
+#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
+#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
 
+extern const struct attribute_group *df_v3_6_attr_groups[];
 extern const struct amdgpu_df_funcs df_v3_6_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 78bd4fc07bab..551c77749b2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)  static int soc15_common_hw_init(void *handle)  {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
 
 	/* enable pcie gen2/3 link */
 	soc15_pcie_gen3_enable(adev);
@@ -1079,6 +1080,11 @@ static int soc15_common_hw_init(void *handle)
 	 */
 	if (adev->nbio_funcs->remap_hdp_registers)
 		adev->nbio_funcs->remap_hdp_registers(adev);
+
+	ret = adev->df_funcs->init(adev);
+	if (ret)
+		return ret;
+
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 	/* HW doorbell routing policy: doorbell writing not @@ -1100,6 +1106,8 @@ static int soc15_common_hw_fini(void *handle)
 	if (amdgpu_sriov_vf(adev))
 		xgpu_ai_mailbox_put_irq(adev);
 
+	adev->df_funcs->fini(adev);
+
 	return 0;
 }
 
--
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* RE: [PATCH 2/2] drm/amdgpu: add pmu counters
       [not found]     ` <20190619224943.130022-2-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-19 23:01       ` Kim, Jonathan
  2019-06-19 23:23       ` Kuehling, Felix
  1 sibling, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-19 23:01 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org,
	Kuehling, Felix

Patch 2 for Felix.

Jon

-----Original Message-----
From: Kim, Jonathan <Jonathan.Kim@amd.com> 
Sent: Wednesday, June 19, 2019 6:50 PM
To: amd-gfx@lists.freedesktop.org
Cc: Kim, Jonathan <Jonathan.Kim@amd.com>
Subject: [PATCH 2/2] drm/amdgpu: add pmu counters

adding perf event counters

Change-Id: Ib8800c2d652e3e788872bd721458ac5422859b62
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile        |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c    | 280 +++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h    |  37 +++
 5 files changed, 325 insertions(+), 1 deletion(-)  create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 57ce44cc3226..4c9fd2645f64 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -54,7 +54,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
 	amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \
 	amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
-	amdgpu_vm_sdma.o
+	amdgpu_vm_sdma.o amdgpu_pmu.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 5780e157fd72..90a159079912 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -964,6 +964,7 @@ struct amdgpu_device {
 
 	uint64_t			unique_id;
 	uint64_t			*df_perfmon_config_assign_mask;
+	struct mutex		df_perfmon_lock;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 003350a2d299..bcfabd025f5d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -61,6 +61,7 @@
 
 #include "amdgpu_xgmi.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_pmu.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
@@ -2757,6 +2758,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		return r;
 	}
 
+	r = amdgpu_pmu_init(adev);
+	if (r)
+		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
+
 	return 0;
 
 failed:
@@ -2822,6 +2827,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	amdgpu_debugfs_regs_cleanup(adev);
 	device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
 	amdgpu_ucode_sysfs_fini(adev);
+	amdgpu_pmu_fini(adev);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
new file mode 100644
index 000000000000..73fd70a59c8b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#include <linux/perf_event.h>
+#include <linux/init.h>
+#include "amdgpu.h"
+#include "amdgpu_pmu.h"
+#include "df_v3_6.h"
+
+#define PMU_NAME_SIZE 32
+
+/* record to keep track of pmu entry per pmu type per device */ struct 
+amdgpu_pmu_entry {
+	struct list_head entry;
+	struct amdgpu_device *adev;
+	struct pmu pmu;
+	unsigned int pmu_perf_type;
+};
+
+static LIST_HEAD(amdgpu_pmu_list);
+
+
+/* initialize perf counter */
+static int amdgpu_perf_event_init(struct perf_event *event) {
+	struct hw_perf_event *hwc = &event->hw;
+
+	/* test the event attr type check for PMU enumeration */
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	/* update the hw_perf_event struct with config data */
+	hwc->conf = event->attr.config;
+
+	return 0;
+}
+
+/* start perf counter */
+static void amdgpu_perf_start(struct perf_event *event, int flags) {
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+		return;
+
+	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+	hwc->state = 0;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		if (!(flags & PERF_EF_RELOAD))
+			pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+
+		pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 0);
+		break;
+	default:
+		break;
+	}
+
+	perf_event_update_userpage(event);
+
+}
+
+/* read perf counter */
+static void amdgpu_perf_read(struct perf_event *event) {
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	u64 count, prev;
+
+	do {
+		prev = local64_read(&hwc->prev_count);
+
+		switch (pe->pmu_perf_type) {
+		case PERF_TYPE_AMDGPU_DF:
+			pe->adev->df_funcs->pmc_get_count(pe->adev, hwc->conf,
+							  &count);
+			break;
+		default:
+			count = 0;
+			break;
+		};
+	} while (local64_cmpxchg(&hwc->prev_count, prev, count) != prev);
+
+	local64_add(count - prev, &event->count); }
+
+/* stop perf counter */
+static void amdgpu_perf_stop(struct perf_event *event, int flags) {
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	if (hwc->state & PERF_HES_UPTODATE)
+		return;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 0);
+		break;
+	default:
+		break;
+	};
+
+	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+	hwc->state |= PERF_HES_STOPPED;
+
+	if (hwc->state & PERF_HES_UPTODATE)
+		return;
+
+	amdgpu_perf_read(event);
+	hwc->state |= PERF_HES_UPTODATE;
+}
+
+/* add perf counter  */
+static int amdgpu_perf_add(struct perf_event *event, int flags) {
+	struct hw_perf_event *hwc = &event->hw;
+	int retval;
+
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		retval = pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+		break;
+	default:
+		return 0;
+	};
+
+	if (retval)
+		return retval;
+
+	if (flags & PERF_EF_START)
+		amdgpu_perf_start(event, PERF_EF_RELOAD);
+
+	return retval;
+
+}
+
+/* delete perf counter  */
+static void amdgpu_perf_del(struct perf_event *event, int flags) {
+	struct hw_perf_event *hwc = &event->hw;
+	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+						  struct amdgpu_pmu_entry,
+						  pmu);
+
+	amdgpu_perf_stop(event, PERF_EF_UPDATE);
+
+	switch (pe->pmu_perf_type) {
+	case PERF_TYPE_AMDGPU_DF:
+		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 1);
+		break;
+	default:
+		break;
+	};
+
+	perf_event_update_userpage(event);
+}
+
+/* vega20 pmus */
+
+/* init pmu tracking per pmu type */
+int init_pmu_by_type(struct amdgpu_device *adev,
+		  const struct attribute_group *attr_groups[],
+		  char *pmu_type_name, char *pmu_file_prefix,
+		  unsigned int pmu_perf_type)
+{
+	char pmu_name[PMU_NAME_SIZE];
+	struct amdgpu_pmu_entry *pmu_entry;
+	int ret = 0;
+
+	pmu_entry = kzalloc(sizeof(struct amdgpu_pmu_entry), GFP_KERNEL);
+
+	if (!pmu_entry)
+		return -ENOMEM;
+
+	pmu_entry->adev = adev;
+	pmu_entry->pmu = (struct pmu){
+		.event_init = amdgpu_perf_event_init,
+		.add = amdgpu_perf_add,
+		.del = amdgpu_perf_del,
+		.start = amdgpu_perf_start,
+		.stop = amdgpu_perf_stop,
+		.read = amdgpu_perf_read,
+		.task_ctx_nr = perf_invalid_context,
+	};
+
+	pmu_entry->pmu.attr_groups = attr_groups;
+	pmu_entry->pmu_perf_type = pmu_perf_type;
+	snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d",
+				pmu_file_prefix, adev->ddev->primary->index);
+
+	ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1);
+
+	if (ret) {
+		kfree(pmu_entry);
+		pr_warn("Error initializing AMDGPU %s PMUs.\n", pmu_type_name);
+		return ret;
+	}
+
+	pr_info("Detected AMDGPU %s Counters. # of Counters = %d.\n",
+			pmu_type_name, DF_V3_6_MAX_COUNTERS);
+
+	list_add_tail(&pmu_entry->entry, &amdgpu_pmu_list);
+
+	return 0;
+}
+
+/* init amdgpu_pmu */
+int amdgpu_pmu_init(struct amdgpu_device *adev) {
+
+	int ret = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		/* init df */
+		ret = init_pmu_by_type(adev, df_v3_6_attr_groups,
+				"DF", "amdgpu_df", PERF_TYPE_AMDGPU_DF);
+
+		/* other pmu types go here*/
+		break;
+	default:
+		return 0;
+	}
+
+	return 0;
+
+}
+
+
+/* destroy all pmu data associated with target device */ void 
+amdgpu_pmu_fini(struct amdgpu_device *adev) {
+	struct amdgpu_pmu_entry *pe, *temp;
+
+	list_for_each_entry_safe(pe, temp, &amdgpu_pmu_list, entry) {
+		if (pe->adev == adev) {
+			list_del(&pe->entry);
+			perf_pmu_unregister(&temp->pmu);
+			kfree(temp);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
new file mode 100644
index 000000000000..7dddb7160a11
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#ifndef _AMDGPU_PMU_H_
+#define _AMDGPU_PMU_H_
+
+enum amdgpu_pmu_perf_type {
+	PERF_TYPE_AMDGPU_DF = 0,
+	PERF_TYPE_AMDGPU_MAX
+};
+
+int amdgpu_pmu_init(struct amdgpu_device *adev); void 
+amdgpu_pmu_fini(struct amdgpu_device *adev);
+
+#endif /* _AMDGPU_PMU_H_ */
--
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* Re: [PATCH 2/2] drm/amdgpu: add pmu counters
       [not found]     ` <20190619224943.130022-2-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-19 23:01       ` Kim, Jonathan
@ 2019-06-19 23:23       ` Kuehling, Felix
  1 sibling, 0 replies; 15+ messages in thread
From: Kuehling, Felix @ 2019-06-19 23:23 UTC (permalink / raw)
  To: Kim, Jonathan,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

Found three more problems. With those fixed, this patch is Reviewed-by: 
Felix Kuehling <Felix.Kuehling@amd.com>

On 2019-06-19 6:50 p.m., Kim, Jonathan wrote:
> adding perf event counters
>
> Change-Id: Ib8800c2d652e3e788872bd721458ac5422859b62
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |   2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   1 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   6 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c    | 280 +++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h    |  37 +++
>   5 files changed, 325 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 57ce44cc3226..4c9fd2645f64 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -54,7 +54,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
>   	amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \
>   	amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
> -	amdgpu_vm_sdma.o
> +	amdgpu_vm_sdma.o amdgpu_pmu.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 5780e157fd72..90a159079912 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -964,6 +964,7 @@ struct amdgpu_device {
>   
>   	uint64_t			unique_id;
>   	uint64_t			*df_perfmon_config_assign_mask;
> +	struct mutex		df_perfmon_lock;

This mutex isn't used in the latest version.


>   };
>   
>   static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 003350a2d299..bcfabd025f5d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -61,6 +61,7 @@
>   
>   #include "amdgpu_xgmi.h"
>   #include "amdgpu_ras.h"
> +#include "amdgpu_pmu.h"
>   
>   MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
>   MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
> @@ -2757,6 +2758,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>   		return r;
>   	}
>   
> +	r = amdgpu_pmu_init(adev);
> +	if (r)
> +		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
> +
>   	return 0;
>   
>   failed:
> @@ -2822,6 +2827,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
>   	amdgpu_debugfs_regs_cleanup(adev);
>   	device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
>   	amdgpu_ucode_sysfs_fini(adev);
> +	amdgpu_pmu_fini(adev);
>   }
>   
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
> new file mode 100644
> index 000000000000..73fd70a59c8b
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
> @@ -0,0 +1,280 @@
> +/*
> + * Copyright 2019 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Author: Jonathan Kim <jonathan.kim@amd.com>
> + *
> + */
> +
> +#include <linux/perf_event.h>
> +#include <linux/init.h>
> +#include "amdgpu.h"
> +#include "amdgpu_pmu.h"
> +#include "df_v3_6.h"
> +
> +#define PMU_NAME_SIZE 32
> +
> +/* record to keep track of pmu entry per pmu type per device */
> +struct amdgpu_pmu_entry {
> +	struct list_head entry;
> +	struct amdgpu_device *adev;
> +	struct pmu pmu;
> +	unsigned int pmu_perf_type;
> +};
> +
> +static LIST_HEAD(amdgpu_pmu_list);
> +
> +
> +/* initialize perf counter */
> +static int amdgpu_perf_event_init(struct perf_event *event)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +
> +	/* test the event attr type check for PMU enumeration */
> +	if (event->attr.type != event->pmu->type)
> +		return -ENOENT;
> +
> +	/* update the hw_perf_event struct with config data */
> +	hwc->conf = event->attr.config;
> +
> +	return 0;
> +}
> +
> +/* start perf counter */
> +static void amdgpu_perf_start(struct perf_event *event, int flags)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
> +						  struct amdgpu_pmu_entry,
> +						  pmu);
> +
> +	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
> +		return;
> +
> +	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
> +	hwc->state = 0;
> +
> +	switch (pe->pmu_perf_type) {
> +	case PERF_TYPE_AMDGPU_DF:
> +		if (!(flags & PERF_EF_RELOAD))
> +			pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
> +
> +		pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 0);
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	perf_event_update_userpage(event);
> +
> +}
> +
> +/* read perf counter */
> +static void amdgpu_perf_read(struct perf_event *event)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
> +						  struct amdgpu_pmu_entry,
> +						  pmu);
> +
> +	u64 count, prev;
> +
> +	do {
> +		prev = local64_read(&hwc->prev_count);
> +
> +		switch (pe->pmu_perf_type) {
> +		case PERF_TYPE_AMDGPU_DF:
> +			pe->adev->df_funcs->pmc_get_count(pe->adev, hwc->conf,
> +							  &count);
> +			break;
> +		default:
> +			count = 0;
> +			break;
> +		};
> +	} while (local64_cmpxchg(&hwc->prev_count, prev, count) != prev);
> +
> +	local64_add(count - prev, &event->count);
> +}
> +
> +/* stop perf counter */
> +static void amdgpu_perf_stop(struct perf_event *event, int flags)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
> +						  struct amdgpu_pmu_entry,
> +						  pmu);
> +
> +	if (hwc->state & PERF_HES_UPTODATE)
> +		return;
> +
> +	switch (pe->pmu_perf_type) {
> +	case PERF_TYPE_AMDGPU_DF:
> +		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 0);
> +		break;
> +	default:
> +		break;
> +	};
> +
> +	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
> +	hwc->state |= PERF_HES_STOPPED;
> +
> +	if (hwc->state & PERF_HES_UPTODATE)
> +		return;
> +
> +	amdgpu_perf_read(event);
> +	hwc->state |= PERF_HES_UPTODATE;
> +}
> +
> +/* add perf counter  */
> +static int amdgpu_perf_add(struct perf_event *event, int flags)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +	int retval;
> +
> +	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
> +						  struct amdgpu_pmu_entry,
> +						  pmu);
> +
> +	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
> +
> +	switch (pe->pmu_perf_type) {
> +	case PERF_TYPE_AMDGPU_DF:
> +		retval = pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
> +		break;
> +	default:
> +		return 0;
> +	};
> +
> +	if (retval)
> +		return retval;
> +
> +	if (flags & PERF_EF_START)
> +		amdgpu_perf_start(event, PERF_EF_RELOAD);
> +
> +	return retval;
> +
> +}
> +
> +/* delete perf counter  */
> +static void amdgpu_perf_del(struct perf_event *event, int flags)
> +{
> +	struct hw_perf_event *hwc = &event->hw;
> +	struct amdgpu_pmu_entry *pe = container_of(event->pmu,
> +						  struct amdgpu_pmu_entry,
> +						  pmu);
> +
> +	amdgpu_perf_stop(event, PERF_EF_UPDATE);
> +
> +	switch (pe->pmu_perf_type) {
> +	case PERF_TYPE_AMDGPU_DF:
> +		pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 1);
> +		break;
> +	default:
> +		break;
> +	};
> +
> +	perf_event_update_userpage(event);
> +}
> +
> +/* vega20 pmus */
> +
> +/* init pmu tracking per pmu type */
> +int init_pmu_by_type(struct amdgpu_device *adev,
> +		  const struct attribute_group *attr_groups[],
> +		  char *pmu_type_name, char *pmu_file_prefix,
> +		  unsigned int pmu_perf_type)
> +{
> +	char pmu_name[PMU_NAME_SIZE];
> +	struct amdgpu_pmu_entry *pmu_entry;
> +	int ret = 0;
> +
> +	pmu_entry = kzalloc(sizeof(struct amdgpu_pmu_entry), GFP_KERNEL);
> +
> +	if (!pmu_entry)
> +		return -ENOMEM;
> +
> +	pmu_entry->adev = adev;
> +	pmu_entry->pmu = (struct pmu){
> +		.event_init = amdgpu_perf_event_init,
> +		.add = amdgpu_perf_add,
> +		.del = amdgpu_perf_del,
> +		.start = amdgpu_perf_start,
> +		.stop = amdgpu_perf_stop,
> +		.read = amdgpu_perf_read,
> +		.task_ctx_nr = perf_invalid_context,
> +	};
> +
> +	pmu_entry->pmu.attr_groups = attr_groups;
> +	pmu_entry->pmu_perf_type = pmu_perf_type;
> +	snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d",
> +				pmu_file_prefix, adev->ddev->primary->index);
> +
> +	ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1);
> +
> +	if (ret) {
> +		kfree(pmu_entry);
> +		pr_warn("Error initializing AMDGPU %s PMUs.\n", pmu_type_name);
> +		return ret;
> +	}
> +
> +	pr_info("Detected AMDGPU %s Counters. # of Counters = %d.\n",
> +			pmu_type_name, DF_V3_6_MAX_COUNTERS);

DF_V3_6_MAX_COUNTERS shouldn't be used here, because this function may 
later handle other types of performance counters.


> +
> +	list_add_tail(&pmu_entry->entry, &amdgpu_pmu_list);
> +
> +	return 0;
> +}
> +
> +/* init amdgpu_pmu */
> +int amdgpu_pmu_init(struct amdgpu_device *adev)
> +{
> +
> +	int ret = 0;
> +
> +	switch (adev->asic_type) {
> +	case CHIP_VEGA20:
> +		/* init df */
> +		ret = init_pmu_by_type(adev, df_v3_6_attr_groups,
> +				"DF", "amdgpu_df", PERF_TYPE_AMDGPU_DF);
> +
> +		/* other pmu types go here*/
> +		break;
> +	default:
> +		return 0;
> +	}
> +
> +	return 0;
> +
> +}
> +
> +
> +/* destroy all pmu data associated with target device */
> +void amdgpu_pmu_fini(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_pmu_entry *pe, *temp;
> +
> +	list_for_each_entry_safe(pe, temp, &amdgpu_pmu_list, entry) {
> +		if (pe->adev == adev) {
> +			list_del(&pe->entry);
> +			perf_pmu_unregister(&temp->pmu);
> +			kfree(temp);

temp is for internal use by list_for_each_entry_safe. For unregistering 
and freeing, you should use pe.

Regards,
   Felix


> +		}
> +	}
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
> new file mode 100644
> index 000000000000..7dddb7160a11
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
> @@ -0,0 +1,37 @@
> +/*
> + * Copyright 2019 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Author: Jonathan Kim <jonathan.kim@amd.com>
> + *
> + */
> +
> +#ifndef _AMDGPU_PMU_H_
> +#define _AMDGPU_PMU_H_
> +
> +enum amdgpu_pmu_perf_type {
> +	PERF_TYPE_AMDGPU_DF = 0,
> +	PERF_TYPE_AMDGPU_MAX
> +};
> +
> +int amdgpu_pmu_init(struct amdgpu_device *adev);
> +void amdgpu_pmu_fini(struct amdgpu_device *adev);
> +
> +#endif /* _AMDGPU_PMU_H_ */
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found] ` <20190619224943.130022-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-19 22:50   ` [PATCH 2/2] drm/amdgpu: add pmu counters Kim, Jonathan
  2019-06-19 23:00   ` [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons Kim, Jonathan
@ 2019-06-19 23:27   ` Kuehling, Felix
       [not found]     ` <6cb35787-eff6-a354-3463-2fe3dd97c9f1-5C7GfCeVMHo@public.gmane.org>
  2 siblings, 1 reply; 15+ messages in thread
From: Kuehling, Felix @ 2019-06-19 23:27 UTC (permalink / raw)
  To: Kim, Jonathan,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

One more possible simplification below.

On 2019-06-19 6:50 p.m., Kim, Jonathan wrote:
> v4: fixed kzalloc error check and modified df func init to return error code
>
> v3: fixed cleanup by adding fini to free up adev df config counters
>
> v2: simplified by removing xgmi references in function names and moving to
> generic df function names.  fixed issue by removing hardcoded cake tx data
> events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
> error code.
>
> add pmu attribute groups and structures for perf events.
> add sysfs to track available df perfmon counters
> fix overflow handling in perfmon counter reads.
>
> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  19 +-
>   drivers/gpu/drm/amd/amdgpu/df_v1_7.c |   3 +-
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 466 +++++++++++++--------------
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>   drivers/gpu/drm/amd/amdgpu/soc15.c   |   8 +
>   5 files changed, 254 insertions(+), 261 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index d355e9a09ad1..5780e157fd72 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -681,7 +681,8 @@ struct amdgpu_nbio_funcs {
>   };
>   
>   struct amdgpu_df_funcs {
> -	void (*init)(struct amdgpu_device *adev);
> +	int (*init)(struct amdgpu_device *adev);
> +	void (*fini)(struct amdgpu_device *adev);
>   	void (*enable_broadcast_mode)(struct amdgpu_device *adev,
>   				      bool enable);
>   	u32 (*get_fb_channel_number)(struct amdgpu_device *adev);
> @@ -962,6 +963,7 @@ struct amdgpu_device {
>   	long				compute_timeout;
>   
>   	uint64_t			unique_id;
> +	uint64_t			*df_perfmon_config_assign_mask;
>   };
>   
>   static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
> @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>   #endif
>   
>   #include "amdgpu_object.h"
> +
> +/* used by df_v3_6.c and amdgpu_pmu.c */
> +#define AMDGPU_PMU_ATTR(_name, _object)				\
> +static ssize_t								\
> +_name##_show(struct device *dev,					\
> +			       struct device_attribute *attr,		\
> +			       char *page)				\
> +{									\
> +	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
> +	return sprintf(page, _object "\n");				\
> +}									\
> +									\
> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
> +
>   #endif
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> index 9935371db7ce..cc964d52dee4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> @@ -29,8 +29,9 @@
>   
>   static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
>   
> -static void df_v1_7_init (struct amdgpu_device *adev)
> +static int df_v1_7_init (struct amdgpu_device *adev)
>   {
> +	return 0;
>   }
>   
>   static void df_v1_7_enable_broadcast_mode(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> index 8c09bf994acd..cae66bc59986 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> @@ -30,8 +30,118 @@
>   static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>   				       16, 32, 0, 0, 0, 2, 4, 8};
>   
> -static void df_v3_6_init(struct amdgpu_device *adev)
> +/* init df format attrs */
> +AMDGPU_PMU_ATTR(event,		"config:0-7");
> +AMDGPU_PMU_ATTR(instance,	"config:8-15");
> +AMDGPU_PMU_ATTR(umask,		"config:16-23");
> +
> +/* df format attributes  */
> +static struct attribute *df_v3_6_format_attrs[] = {
> +	&pmu_attr_event.attr,
> +	&pmu_attr_instance.attr,
> +	&pmu_attr_umask.attr,
> +	NULL
> +};
> +
> +/* df format attribute group */
> +static struct attribute_group df_v3_6_format_attr_group = {
> +	.name = "format",
> +	.attrs = df_v3_6_format_attrs,
> +};
> +
> +/* df event attrs */
> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
> +		      "event=0x7,instance=0x46,umask=0x2");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
> +		      "event=0x7,instance=0x47,umask=0x2");
> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
> +		      "event=0x7,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
> +		      "event=0x7,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x46,umask=0x8");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x47,umask=0x8");
> +
> +/* df event attributes  */
> +static struct attribute *df_v3_6_event_attrs[] = {
> +	&pmu_attr_cake0_pcsout_txdata.attr,
> +	&pmu_attr_cake1_pcsout_txdata.attr,
> +	&pmu_attr_cake0_pcsout_txmeta.attr,
> +	&pmu_attr_cake1_pcsout_txmeta.attr,
> +	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
> +	NULL
> +};
> +
> +/* df event attribute group */
> +static struct attribute_group df_v3_6_event_attr_group = {
> +	.name = "events",
> +	.attrs = df_v3_6_event_attrs
> +};
> +
> +/* df event attr groups  */
> +const struct attribute_group *df_v3_6_attr_groups[] = {
> +		&df_v3_6_format_attr_group,
> +		&df_v3_6_event_attr_group,
> +		NULL
> +};
> +
> +/* get the number of df counters available */
> +static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
> +		struct device_attribute *attr,
> +		char *buf)
> +{
> +	struct amdgpu_device *adev;
> +	struct drm_device *ddev;
> +	int i, count;
> +
> +	ddev = dev_get_drvdata(dev);
> +	adev = ddev->dev_private;
> +	count = 0;
> +
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0)
> +			count++;
> +	}
> +
> +	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
> +}
> +
> +/* device attr for available perfmon counters */
> +static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
> +
> +/* init perfmons */
> +static int df_v3_6_init(struct amdgpu_device *adev)
> +{
> +	int i, ret;
> +
> +	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
> +	if (ret)
> +		DRM_ERROR("failed to create file for available df counters\n");
> +
> +	adev->df_perfmon_config_assign_mask =
> +		kzalloc(sizeof(uint64_t) * DF_V3_6_MAX_COUNTERS, GFP_KERNEL);

The size of the array is fixed and quite small (only 4 entries). Maybe 
we don't need to allocate this dynamically at all. Just make it a fixed 
size array in adev. Then this function can't fail, which simplifies the 
error handling in soc15_common_hw_init and you don't need to add the 
_fini function to free the memory any more.


> +
> +	if (!adev->df_perfmon_config_assign_mask)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++)
> +		adev->df_perfmon_config_assign_mask[i] = 0;
> +
> +	return 0;
> +}
> +
> +static void df_v3_6_fini(struct amdgpu_device *adev)
>   {
> +	kfree(adev->df_perfmon_config_assign_mask);
> +	adev->df_perfmon_config_assign_mask = NULL;
>   }
>   
>   static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
> @@ -105,28 +215,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>   		*flags |= AMD_CG_SUPPORT_DF_MGCG;
>   }
>   
> -/* hold counter assignment per gpu struct */
> -struct df_v3_6_event_mask {
> -		struct amdgpu_device gpu;
> -		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
> -};
> -
>   /* get assigned df perfmon ctr as int */
> -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> -				      uint64_t config,
> -				      int *counter)
> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
> -	struct df_v3_6_event_mask *mask;
>   	int i;
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
> -			*counter = i;
> -			return;
> -		}
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if ((config & 0x0FFFFFFUL) ==
> +					adev->df_perfmon_config_assign_mask[i])
> +			return i;
>   	}
> +
> +	return -EINVAL;
>   }
>   
>   /* get address based on counter assignment */
> @@ -136,10 +237,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>   				 uint32_t *lo_base_addr,
>   				 uint32_t *hi_base_addr)
>   {
> -
> -	int target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr < 0)
>   		return;
> @@ -177,40 +275,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>   }
>   
>   /* get control counter settings i.e. address and values to set */
> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>   					  uint64_t config,
>   					  uint32_t *lo_base_addr,
>   					  uint32_t *hi_base_addr,
>   					  uint32_t *lo_val,
>   					  uint32_t *hi_val)
>   {
> -
> -	uint32_t eventsel, instance, unitmask;
> -	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
> -
>   	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
>   
> -	if (lo_val == NULL || hi_val == NULL)
> -		return;
> -
>   	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
> -		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
> +		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>   				*lo_base_addr, *hi_base_addr);
> -		return;
> +		return -ENXIO;
>   	}
>   
> -	eventsel = GET_EVENT(config);
> -	instance = GET_INSTANCE(config);
> -	unitmask = GET_UNITMASK(config);
> -
> -	es_5_0 = eventsel & 0x3FUL;
> -	es_13_6 = instance;
> -	es_13_0 = (es_13_6 << 6) + es_5_0;
> -	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
> -	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
> -	es_7_0 = es_13_0 & 0x0FFUL;
> -	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
> -	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
> +	if (lo_val && hi_val) {
> +		uint32_t eventsel, instance, unitmask;
> +		uint32_t instance_10, instance_5432, instance_76;
> +
> +		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
> +		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
> +		instance = DF_V3_6_GET_INSTANCE(config);
> +
> +		instance_10 = instance & 0x3;
> +		instance_5432 = (instance >> 2) & 0xf;
> +		instance_76 = (instance >> 6) & 0x3;
> +
> +		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
> +		*hi_val = (instance_76 << 29) | instance_5432;
> +	}
> +
> +	return 0;
>   }
>   
>   /* assign df performance counters for read */
> @@ -218,26 +314,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   				   uint64_t config,
>   				   int *is_assigned)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
>   	int i, target_cntr;
>   
> -	target_cntr = -1;
> -
>   	*is_assigned = 0;
>   
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0) {
>   		*is_assigned = 1;
>   		return 0;
>   	}
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if (mask->config_assign_mask[i] == 0ULL) {
> -			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
> +			adev->df_perfmon_config_assign_mask[i] =
> +							config & 0x0FFFFFFUL;
>   			return 0;
>   		}
>   	}
> @@ -249,66 +340,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>   				     uint64_t config)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
> -	int target_cntr;
> -
> -	target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> -
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0)
> -		mask->config_assign_mask[target_cntr] = 0ULL;
> -
> +		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>   }
>   
> -/*
> - * get xgmi link counters via programmable data fabric (df) counters (max 4)
> - * using cake tx event.
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @count -> counters to pass
> - *
> - */
>   
> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       uint64_t *count)
> -{
> -	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> -				      &hi_base_addr);
> -
> -	if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -		return;
> -
> -	lo_val = RREG32_PCIE(lo_base_addr);
> -	hi_val = RREG32_PCIE(hi_base_addr);
> -
> -	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> -}
> -
> -/*
> - * reset xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
> -					 int instance)
> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
> +					 uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr;
> -	uint64_t config;
> -
> -	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>   
>   	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>   				      &hi_base_addr);
> @@ -320,185 +362,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>   	WREG32_PCIE(hi_base_addr, 0UL);
>   }
>   
> -/*
> - * add xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
>   
> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
> -				      int instance)
> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
>   	int ret, is_assigned;
>   
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
>   	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>   
>   	if (ret || is_assigned)
>   		return ret;
>   
> -	df_v3_6_pmc_get_ctrl_settings(adev,
> +	ret = df_v3_6_pmc_get_ctrl_settings(adev,
>   			config,
>   			&lo_base_addr,
>   			&hi_base_addr,
>   			&lo_val,
>   			&hi_val);
>   
> +	if (ret)
> +		return ret;
> +
> +	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
> +
>   	WREG32_PCIE(lo_base_addr, lo_val);
>   	WREG32_PCIE(hi_base_addr, hi_val);
>   
>   	return ret;
>   }
>   
> -
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either resume or assign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
> -					int instance,
> -					int is_enable)
> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> +			     int is_enable)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -	int ret;
> -
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> +	int ret = 0;
>   
> -	if (is_enable) {
> -
> -		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
> -
> -		if (ret)
> -			return ret;
> +	switch (adev->asic_type) {
> +	case CHIP_VEGA20:
>   
> -	} else {
> +		df_v3_6_reset_perfmon_cntr(adev, config);
>   
> -		config = GET_INSTANCE_CONFIG(instance);
> +		if (is_enable) {
> +			ret = df_v3_6_add_perfmon_cntr(adev, config);
> +		} else {
> +			ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +					config,
> +					&lo_base_addr,
> +					&hi_base_addr,
> +					NULL,
> +					NULL);
>   
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> +			if (ret)
> +				return ret;
>   
> -		if (lo_base_addr == 0)
> -			return -EINVAL;
> +			lo_val = RREG32_PCIE(lo_base_addr);
>   
> -		lo_val = RREG32_PCIE(lo_base_addr);
> +			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +		}
>   
> -		ret = 0;
> +		break;
> +	default:
> +		break;
>   	}
>   
>   	return ret;
> -
>   }
>   
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either pause or unassign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       int is_disable)
> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> +			    int is_disable)
>   {
> -
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	if (is_disable) {
> -		df_v3_6_reset_xgmi_link_cntr(adev, instance);
> -		df_v3_6_pmc_release_cntr(adev, config);
> -	} else {
> -
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> -
> -		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -			return -EINVAL;
> -
> -		lo_val = RREG32_PCIE(lo_base_addr);
> -
> -		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
> -	}
> -
> -	return 0;
> -}
> -
> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> -			     int is_enable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +	int ret = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -		if (xgmi_tx_link >= 0)
> -			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
> -						      is_enable);
> +		ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +			config,
> +			&lo_base_addr,
> +			&hi_base_addr,
> +			NULL,
> +			NULL);
>   
>   		if (ret)
>   			return ret;
>   
> -		ret = 0;
> -		break;
> -	default:
> -		break;
> -	}
> +		lo_val = RREG32_PCIE(lo_base_addr);
>   
> -	return ret;
> -}
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> -			    int is_disable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>   
> -	switch (adev->asic_type) {
> -	case CHIP_VEGA20:
> -			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -			if (xgmi_tx_link >= 0) {
> -				ret = df_v3_6_stop_xgmi_link_cntr(adev,
> -								  xgmi_tx_link,
> -								  is_disable);
> -				if (ret)
> -					return ret;
> -			}
> -
> -			ret = 0;
> -			break;
> +		if (is_disable)
> +			df_v3_6_pmc_release_cntr(adev, config);
> +
> +		break;
>   	default:
>   		break;
>   	}
> @@ -510,28 +473,39 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>   				  uint64_t config,
>   				  uint64_t *count)
>   {
> -
> -	int xgmi_tx_link;
> +	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> +	*count = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>   
> -		if (xgmi_tx_link >= 0) {
> -			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
> -			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
> -		}
> +		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> +				      &hi_base_addr);
> +
> +		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> +			return;
> +
> +		lo_val = RREG32_PCIE(lo_base_addr);
> +		hi_val = RREG32_PCIE(hi_base_addr);
> +
> +		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> +
> +		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
> +			*count = 0;
> +
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>   
>   		break;
> +
>   	default:
>   		break;
>   	}
> -
>   }
>   
>   const struct amdgpu_df_funcs df_v3_6_funcs = {
>   	.init = df_v3_6_init,
> +	.fini = df_v3_6_fini,
>   	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
>   	.get_fb_channel_number = df_v3_6_get_fb_channel_number,
>   	.get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> index fcffd807764d..76998541bc30 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>   };
>   
>   /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
> -#define AMDGPU_DF_MAX_COUNTERS		4
> +#define DF_V3_6_MAX_COUNTERS		4
>   
>   /* get flags from df perfmon config */
> -#define GET_EVENT(x)			(x & 0xFFUL)
> -#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
> -#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
> -#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
> -					| ((0x046ULL + x) << 8) \
> -					| (0x02 << 16))
> -
> -/* df event conf macros */
> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
> +#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
> +#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
> +#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
> +#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
>   
> +extern const struct attribute_group *df_v3_6_attr_groups[];
>   extern const struct amdgpu_df_funcs df_v3_6_funcs;
>   
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 78bd4fc07bab..551c77749b2d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>   static int soc15_common_hw_init(void *handle)
>   {
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +	int ret;
>   
>   	/* enable pcie gen2/3 link */
>   	soc15_pcie_gen3_enable(adev);
> @@ -1079,6 +1080,11 @@ static int soc15_common_hw_init(void *handle)
>   	 */
>   	if (adev->nbio_funcs->remap_hdp_registers)
>   		adev->nbio_funcs->remap_hdp_registers(adev);
> +
> +	ret = adev->df_funcs->init(adev);
> +	if (ret)
> +		return ret;
> +
>   	/* enable the doorbell aperture */
>   	soc15_enable_doorbell_aperture(adev, true);
>   	/* HW doorbell routing policy: doorbell writing not
> @@ -1100,6 +1106,8 @@ static int soc15_common_hw_fini(void *handle)
>   	if (amdgpu_sriov_vf(adev))
>   		xgpu_ai_mailbox_put_irq(adev);
>   
> +	adev->df_funcs->fini(adev);
> +
>   	return 0;
>   }
>   
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* RE: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found]     ` <6cb35787-eff6-a354-3463-2fe3dd97c9f1-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-19 23:59       ` Kim, Jonathan
  0 siblings, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-19 23:59 UTC (permalink / raw)
  To: Kuehling, Felix,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

I did dynamic allocation in case future df versions change the number of perfmons available.
I could set a fixed hard limit (like 16) in adev since the this version of df code uses DF_V3_6_MAX_COUNTERS to manage counters availability.
And if we exceed 16 for future ips, then do a re-review or just up the hard limit if it's still small enough.

Jon

-----Original Message-----
From: Kuehling, Felix <Felix.Kuehling@amd.com> 
Sent: Wednesday, June 19, 2019 7:28 PM
To: Kim, Jonathan <Jonathan.Kim@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons

One more possible simplification below.

On 2019-06-19 6:50 p.m., Kim, Jonathan wrote:
> v4: fixed kzalloc error check and modified df func init to return 
> error code
>
> v3: fixed cleanup by adding fini to free up adev df config counters
>
> v2: simplified by removing xgmi references in function names and 
> moving to generic df function names.  fixed issue by removing 
> hardcoded cake tx data events. streamlined error handling by having  
> df_v3_6_pmc_get_ctrl return error code.
>
> add pmu attribute groups and structures for perf events.
> add sysfs to track available df perfmon counters fix overflow handling 
> in perfmon counter reads.
>
> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  19 +-
>   drivers/gpu/drm/amd/amdgpu/df_v1_7.c |   3 +-
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 466 +++++++++++++--------------
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>   drivers/gpu/drm/amd/amdgpu/soc15.c   |   8 +
>   5 files changed, 254 insertions(+), 261 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index d355e9a09ad1..5780e157fd72 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -681,7 +681,8 @@ struct amdgpu_nbio_funcs {
>   };
>   
>   struct amdgpu_df_funcs {
> -	void (*init)(struct amdgpu_device *adev);
> +	int (*init)(struct amdgpu_device *adev);
> +	void (*fini)(struct amdgpu_device *adev);
>   	void (*enable_broadcast_mode)(struct amdgpu_device *adev,
>   				      bool enable);
>   	u32 (*get_fb_channel_number)(struct amdgpu_device *adev); @@ -962,6 
> +963,7 @@ struct amdgpu_device {
>   	long				compute_timeout;
>   
>   	uint64_t			unique_id;
> +	uint64_t			*df_perfmon_config_assign_mask;
>   };
>   
>   static inline struct amdgpu_device *amdgpu_ttm_adev(struct 
> ttm_bo_device *bdev) @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>   #endif
>   
>   #include "amdgpu_object.h"
> +
> +/* used by df_v3_6.c and amdgpu_pmu.c */
> +#define AMDGPU_PMU_ATTR(_name, _object)				\
> +static ssize_t								\
> +_name##_show(struct device *dev,					\
> +			       struct device_attribute *attr,		\
> +			       char *page)				\
> +{									\
> +	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
> +	return sprintf(page, _object "\n");				\
> +}									\
> +									\
> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
> +
>   #endif
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c 
> b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> index 9935371db7ce..cc964d52dee4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
> @@ -29,8 +29,9 @@
>   
>   static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
>   
> -static void df_v1_7_init (struct amdgpu_device *adev)
> +static int df_v1_7_init (struct amdgpu_device *adev)
>   {
> +	return 0;
>   }
>   
>   static void df_v1_7_enable_broadcast_mode(struct amdgpu_device 
> *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c 
> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> index 8c09bf994acd..cae66bc59986 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> @@ -30,8 +30,118 @@
>   static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>   				       16, 32, 0, 0, 0, 2, 4, 8};
>   
> -static void df_v3_6_init(struct amdgpu_device *adev)
> +/* init df format attrs */
> +AMDGPU_PMU_ATTR(event,		"config:0-7");
> +AMDGPU_PMU_ATTR(instance,	"config:8-15");
> +AMDGPU_PMU_ATTR(umask,		"config:16-23");
> +
> +/* df format attributes  */
> +static struct attribute *df_v3_6_format_attrs[] = {
> +	&pmu_attr_event.attr,
> +	&pmu_attr_instance.attr,
> +	&pmu_attr_umask.attr,
> +	NULL
> +};
> +
> +/* df format attribute group */
> +static struct attribute_group df_v3_6_format_attr_group = {
> +	.name = "format",
> +	.attrs = df_v3_6_format_attrs,
> +};
> +
> +/* df event attrs */
> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
> +		      "event=0x7,instance=0x46,umask=0x2");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
> +		      "event=0x7,instance=0x47,umask=0x2");
> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
> +		      "event=0x7,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
> +		      "event=0x7,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x46,umask=0x8");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x47,umask=0x8");
> +
> +/* df event attributes  */
> +static struct attribute *df_v3_6_event_attrs[] = {
> +	&pmu_attr_cake0_pcsout_txdata.attr,
> +	&pmu_attr_cake1_pcsout_txdata.attr,
> +	&pmu_attr_cake0_pcsout_txmeta.attr,
> +	&pmu_attr_cake1_pcsout_txmeta.attr,
> +	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
> +	NULL
> +};
> +
> +/* df event attribute group */
> +static struct attribute_group df_v3_6_event_attr_group = {
> +	.name = "events",
> +	.attrs = df_v3_6_event_attrs
> +};
> +
> +/* df event attr groups  */
> +const struct attribute_group *df_v3_6_attr_groups[] = {
> +		&df_v3_6_format_attr_group,
> +		&df_v3_6_event_attr_group,
> +		NULL
> +};
> +
> +/* get the number of df counters available */ static ssize_t 
> +df_v3_6_get_df_cntr_avail(struct device *dev,
> +		struct device_attribute *attr,
> +		char *buf)
> +{
> +	struct amdgpu_device *adev;
> +	struct drm_device *ddev;
> +	int i, count;
> +
> +	ddev = dev_get_drvdata(dev);
> +	adev = ddev->dev_private;
> +	count = 0;
> +
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0)
> +			count++;
> +	}
> +
> +	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
> +}
> +
> +/* device attr for available perfmon counters */ static 
> +DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
> +
> +/* init perfmons */
> +static int df_v3_6_init(struct amdgpu_device *adev) {
> +	int i, ret;
> +
> +	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
> +	if (ret)
> +		DRM_ERROR("failed to create file for available df counters\n");
> +
> +	adev->df_perfmon_config_assign_mask =
> +		kzalloc(sizeof(uint64_t) * DF_V3_6_MAX_COUNTERS, GFP_KERNEL);

The size of the array is fixed and quite small (only 4 entries). Maybe we don't need to allocate this dynamically at all. Just make it a fixed size array in adev. Then this function can't fail, which simplifies the error handling in soc15_common_hw_init and you don't need to add the _fini function to free the memory any more.


> +
> +	if (!adev->df_perfmon_config_assign_mask)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++)
> +		adev->df_perfmon_config_assign_mask[i] = 0;
> +
> +	return 0;
> +}
> +
> +static void df_v3_6_fini(struct amdgpu_device *adev)
>   {
> +	kfree(adev->df_perfmon_config_assign_mask);
> +	adev->df_perfmon_config_assign_mask = NULL;
>   }
>   
>   static void df_v3_6_enable_broadcast_mode(struct amdgpu_device 
> *adev, @@ -105,28 +215,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>   		*flags |= AMD_CG_SUPPORT_DF_MGCG;
>   }
>   
> -/* hold counter assignment per gpu struct */ -struct 
> df_v3_6_event_mask {
> -		struct amdgpu_device gpu;
> -		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
> -};
> -
>   /* get assigned df perfmon ctr as int */ -static void 
> df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> -				      uint64_t config,
> -				      int *counter)
> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
> -	struct df_v3_6_event_mask *mask;
>   	int i;
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
> -			*counter = i;
> -			return;
> -		}
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if ((config & 0x0FFFFFFUL) ==
> +					adev->df_perfmon_config_assign_mask[i])
> +			return i;
>   	}
> +
> +	return -EINVAL;
>   }
>   
>   /* get address based on counter assignment */ @@ -136,10 +237,7 @@ 
> static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>   				 uint32_t *lo_base_addr,
>   				 uint32_t *hi_base_addr)
>   {
> -
> -	int target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr < 0)
>   		return;
> @@ -177,40 +275,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>   }
>   
>   /* get control counter settings i.e. address and values to set */ 
> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>   					  uint64_t config,
>   					  uint32_t *lo_base_addr,
>   					  uint32_t *hi_base_addr,
>   					  uint32_t *lo_val,
>   					  uint32_t *hi_val)
>   {
> -
> -	uint32_t eventsel, instance, unitmask;
> -	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
> -
>   	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
>   
> -	if (lo_val == NULL || hi_val == NULL)
> -		return;
> -
>   	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
> -		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
> +		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>   				*lo_base_addr, *hi_base_addr);
> -		return;
> +		return -ENXIO;
>   	}
>   
> -	eventsel = GET_EVENT(config);
> -	instance = GET_INSTANCE(config);
> -	unitmask = GET_UNITMASK(config);
> -
> -	es_5_0 = eventsel & 0x3FUL;
> -	es_13_6 = instance;
> -	es_13_0 = (es_13_6 << 6) + es_5_0;
> -	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
> -	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
> -	es_7_0 = es_13_0 & 0x0FFUL;
> -	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
> -	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
> +	if (lo_val && hi_val) {
> +		uint32_t eventsel, instance, unitmask;
> +		uint32_t instance_10, instance_5432, instance_76;
> +
> +		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
> +		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
> +		instance = DF_V3_6_GET_INSTANCE(config);
> +
> +		instance_10 = instance & 0x3;
> +		instance_5432 = (instance >> 2) & 0xf;
> +		instance_76 = (instance >> 6) & 0x3;
> +
> +		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
> +		*hi_val = (instance_76 << 29) | instance_5432;
> +	}
> +
> +	return 0;
>   }
>   
>   /* assign df performance counters for read */ @@ -218,26 +314,21 @@ 
> static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   				   uint64_t config,
>   				   int *is_assigned)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
>   	int i, target_cntr;
>   
> -	target_cntr = -1;
> -
>   	*is_assigned = 0;
>   
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0) {
>   		*is_assigned = 1;
>   		return 0;
>   	}
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if (mask->config_assign_mask[i] == 0ULL) {
> -			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
> +			adev->df_perfmon_config_assign_mask[i] =
> +							config & 0x0FFFFFFUL;
>   			return 0;
>   		}
>   	}
> @@ -249,66 +340,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>   				     uint64_t config)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
> -	int target_cntr;
> -
> -	target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> -
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0)
> -		mask->config_assign_mask[target_cntr] = 0ULL;
> -
> +		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>   }
>   
> -/*
> - * get xgmi link counters via programmable data fabric (df) counters 
> (max 4)
> - * using cake tx event.
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @count -> counters to pass
> - *
> - */
>   
> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       uint64_t *count)
> -{
> -	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> -				      &hi_base_addr);
> -
> -	if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -		return;
> -
> -	lo_val = RREG32_PCIE(lo_base_addr);
> -	hi_val = RREG32_PCIE(hi_base_addr);
> -
> -	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> -}
> -
> -/*
> - * reset xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
> -					 int instance)
> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
> +					 uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr;
> -	uint64_t config;
> -
> -	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>   
>   	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>   				      &hi_base_addr);
> @@ -320,185 +362,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>   	WREG32_PCIE(hi_base_addr, 0UL);
>   }
>   
> -/*
> - * add xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
>   
> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
> -				      int instance)
> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
>   	int ret, is_assigned;
>   
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
>   	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>   
>   	if (ret || is_assigned)
>   		return ret;
>   
> -	df_v3_6_pmc_get_ctrl_settings(adev,
> +	ret = df_v3_6_pmc_get_ctrl_settings(adev,
>   			config,
>   			&lo_base_addr,
>   			&hi_base_addr,
>   			&lo_val,
>   			&hi_val);
>   
> +	if (ret)
> +		return ret;
> +
> +	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
> +
>   	WREG32_PCIE(lo_base_addr, lo_val);
>   	WREG32_PCIE(hi_base_addr, hi_val);
>   
>   	return ret;
>   }
>   
> -
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either resume or assign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
> -					int instance,
> -					int is_enable)
> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> +			     int is_enable)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -	int ret;
> -
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> +	int ret = 0;
>   
> -	if (is_enable) {
> -
> -		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
> -
> -		if (ret)
> -			return ret;
> +	switch (adev->asic_type) {
> +	case CHIP_VEGA20:
>   
> -	} else {
> +		df_v3_6_reset_perfmon_cntr(adev, config);
>   
> -		config = GET_INSTANCE_CONFIG(instance);
> +		if (is_enable) {
> +			ret = df_v3_6_add_perfmon_cntr(adev, config);
> +		} else {
> +			ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +					config,
> +					&lo_base_addr,
> +					&hi_base_addr,
> +					NULL,
> +					NULL);
>   
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> +			if (ret)
> +				return ret;
>   
> -		if (lo_base_addr == 0)
> -			return -EINVAL;
> +			lo_val = RREG32_PCIE(lo_base_addr);
>   
> -		lo_val = RREG32_PCIE(lo_base_addr);
> +			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +		}
>   
> -		ret = 0;
> +		break;
> +	default:
> +		break;
>   	}
>   
>   	return ret;
> -
>   }
>   
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either pause or unassign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       int is_disable)
> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> +			    int is_disable)
>   {
> -
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	if (is_disable) {
> -		df_v3_6_reset_xgmi_link_cntr(adev, instance);
> -		df_v3_6_pmc_release_cntr(adev, config);
> -	} else {
> -
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> -
> -		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -			return -EINVAL;
> -
> -		lo_val = RREG32_PCIE(lo_base_addr);
> -
> -		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
> -	}
> -
> -	return 0;
> -}
> -
> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> -			     int is_enable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +	int ret = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -		if (xgmi_tx_link >= 0)
> -			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
> -						      is_enable);
> +		ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +			config,
> +			&lo_base_addr,
> +			&hi_base_addr,
> +			NULL,
> +			NULL);
>   
>   		if (ret)
>   			return ret;
>   
> -		ret = 0;
> -		break;
> -	default:
> -		break;
> -	}
> +		lo_val = RREG32_PCIE(lo_base_addr);
>   
> -	return ret;
> -}
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> -			    int is_disable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>   
> -	switch (adev->asic_type) {
> -	case CHIP_VEGA20:
> -			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -			if (xgmi_tx_link >= 0) {
> -				ret = df_v3_6_stop_xgmi_link_cntr(adev,
> -								  xgmi_tx_link,
> -								  is_disable);
> -				if (ret)
> -					return ret;
> -			}
> -
> -			ret = 0;
> -			break;
> +		if (is_disable)
> +			df_v3_6_pmc_release_cntr(adev, config);
> +
> +		break;
>   	default:
>   		break;
>   	}
> @@ -510,28 +473,39 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>   				  uint64_t config,
>   				  uint64_t *count)
>   {
> -
> -	int xgmi_tx_link;
> +	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> +	*count = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>   
> -		if (xgmi_tx_link >= 0) {
> -			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
> -			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
> -		}
> +		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> +				      &hi_base_addr);
> +
> +		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> +			return;
> +
> +		lo_val = RREG32_PCIE(lo_base_addr);
> +		hi_val = RREG32_PCIE(hi_base_addr);
> +
> +		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> +
> +		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
> +			*count = 0;
> +
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>   
>   		break;
> +
>   	default:
>   		break;
>   	}
> -
>   }
>   
>   const struct amdgpu_df_funcs df_v3_6_funcs = {
>   	.init = df_v3_6_init,
> +	.fini = df_v3_6_fini,
>   	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
>   	.get_fb_channel_number = df_v3_6_get_fb_channel_number,
>   	.get_hbm_channel_number = df_v3_6_get_hbm_channel_number, diff 
> --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h 
> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> index fcffd807764d..76998541bc30 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>   };
>   
>   /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
> -#define AMDGPU_DF_MAX_COUNTERS		4
> +#define DF_V3_6_MAX_COUNTERS		4
>   
>   /* get flags from df perfmon config */
> -#define GET_EVENT(x)			(x & 0xFFUL)
> -#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
> -#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
> -#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
> -					| ((0x046ULL + x) << 8) \
> -					| (0x02 << 16))
> -
> -/* df event conf macros */
> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
> +#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
> +#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
> +#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
> +#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
>   
> +extern const struct attribute_group *df_v3_6_attr_groups[];
>   extern const struct amdgpu_df_funcs df_v3_6_funcs;
>   
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c 
> b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 78bd4fc07bab..551c77749b2d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>   static int soc15_common_hw_init(void *handle)
>   {
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +	int ret;
>   
>   	/* enable pcie gen2/3 link */
>   	soc15_pcie_gen3_enable(adev);
> @@ -1079,6 +1080,11 @@ static int soc15_common_hw_init(void *handle)
>   	 */
>   	if (adev->nbio_funcs->remap_hdp_registers)
>   		adev->nbio_funcs->remap_hdp_registers(adev);
> +
> +	ret = adev->df_funcs->init(adev);
> +	if (ret)
> +		return ret;
> +
>   	/* enable the doorbell aperture */
>   	soc15_enable_doorbell_aperture(adev, true);
>   	/* HW doorbell routing policy: doorbell writing not @@ -1100,6 
> +1106,8 @@ static int soc15_common_hw_fini(void *handle)
>   	if (amdgpu_sriov_vf(adev))
>   		xgpu_ai_mailbox_put_irq(adev);
>   
> +	adev->df_funcs->fini(adev);
> +
>   	return 0;
>   }
>   
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
@ 2019-06-20  0:53 Kim, Jonathan
       [not found] ` <20190620005313.56371-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  0 siblings, 1 reply; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-20  0:53 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org; +Cc: Kim, Jonathan

v4: fixed kzalloc error check and modified df func init to return error code

v3: fixed cleanup by adding fini to free up adev df config counters

v2: simplified by removing xgmi references in function names and moving to
generic df function names.  fixed issue by removing hardcoded cake tx data
events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
error code.

add pmu attribute groups and structures for perf events.
add sysfs to track available df perfmon counters
fix overflow handling in perfmon counter reads.

Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +
 drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------
 drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
 4 files changed, 231 insertions(+), 258 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d355e9a09ad1..91cfcc7be5c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -732,6 +732,7 @@ struct amd_powerplay {
 };
 
 #define AMDGPU_RESET_MAGIC_NUM 64
+#define AMDGPU_MAX_DF_PERFMONS 16
 struct amdgpu_device {
 	struct device			*dev;
 	struct drm_device		*ddev;
@@ -962,6 +963,7 @@ struct amdgpu_device {
 	long				compute_timeout;
 
 	uint64_t			unique_id;
+	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
 #endif
 
 #include "amdgpu_object.h"
+
+/* used by df_v3_6.c and amdgpu_pmu.c */
+#define AMDGPU_PMU_ATTR(_name, _object)				\
+static ssize_t								\
+_name##_show(struct device *dev,					\
+			       struct device_attribute *attr,		\
+			       char *page)				\
+{									\
+	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
+	return sprintf(page, _object "\n");				\
+}									\
+									\
+static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
+
 #endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index 8c09bf994acd..12e3e67013d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -30,8 +30,104 @@
 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
 				       16, 32, 0, 0, 0, 2, 4, 8};
 
+/* init df format attrs */
+AMDGPU_PMU_ATTR(event,		"config:0-7");
+AMDGPU_PMU_ATTR(instance,	"config:8-15");
+AMDGPU_PMU_ATTR(umask,		"config:16-23");
+
+/* df format attributes  */
+static struct attribute *df_v3_6_format_attrs[] = {
+	&pmu_attr_event.attr,
+	&pmu_attr_instance.attr,
+	&pmu_attr_umask.attr,
+	NULL
+};
+
+/* df format attribute group */
+static struct attribute_group df_v3_6_format_attr_group = {
+	.name = "format",
+	.attrs = df_v3_6_format_attrs,
+};
+
+/* df event attrs */
+AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
+		      "event=0x7,instance=0x46,umask=0x2");
+AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
+		      "event=0x7,instance=0x47,umask=0x2");
+AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
+		      "event=0x7,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
+		      "event=0x7,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x46,umask=0x8");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x47,umask=0x8");
+
+/* df event attributes  */
+static struct attribute *df_v3_6_event_attrs[] = {
+	&pmu_attr_cake0_pcsout_txdata.attr,
+	&pmu_attr_cake1_pcsout_txdata.attr,
+	&pmu_attr_cake0_pcsout_txmeta.attr,
+	&pmu_attr_cake1_pcsout_txmeta.attr,
+	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
+	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
+	NULL
+};
+
+/* df event attribute group */
+static struct attribute_group df_v3_6_event_attr_group = {
+	.name = "events",
+	.attrs = df_v3_6_event_attrs
+};
+
+/* df event attr groups  */
+const struct attribute_group *df_v3_6_attr_groups[] = {
+		&df_v3_6_format_attr_group,
+		&df_v3_6_event_attr_group,
+		NULL
+};
+
+/* get the number of df counters available */
+static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *ddev;
+	int i, count;
+
+	ddev = dev_get_drvdata(dev);
+	adev = ddev->dev_private;
+	count = 0;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0)
+			count++;
+	}
+
+	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
+}
+
+/* device attr for available perfmon counters */
+static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
+
+/* init perfmons */
 static void df_v3_6_init(struct amdgpu_device *adev)
 {
+	int i, ret;
+
+	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
+	if (ret)
+		DRM_ERROR("failed to create file for available df counters\n");
+
+	for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
+		adev->df_perfmon_config_assign_mask[i] = 0;
 }
 
 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
@@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 		*flags |= AMD_CG_SUPPORT_DF_MGCG;
 }
 
-/* hold counter assignment per gpu struct */
-struct df_v3_6_event_mask {
-		struct amdgpu_device gpu;
-		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
-};
-
 /* get assigned df perfmon ctr as int */
-static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
-				      uint64_t config,
-				      int *counter)
+static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
-	struct df_v3_6_event_mask *mask;
 	int i;
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
-			*counter = i;
-			return;
-		}
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if ((config & 0x0FFFFFFUL) ==
+					adev->df_perfmon_config_assign_mask[i])
+			return i;
 	}
+
+	return -EINVAL;
 }
 
 /* get address based on counter assignment */
@@ -136,10 +223,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 				 uint32_t *lo_base_addr,
 				 uint32_t *hi_base_addr)
 {
-
-	int target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr < 0)
 		return;
@@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
 }
 
 /* get control counter settings i.e. address and values to set */
-static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 					  uint64_t config,
 					  uint32_t *lo_base_addr,
 					  uint32_t *hi_base_addr,
 					  uint32_t *lo_val,
 					  uint32_t *hi_val)
 {
-
-	uint32_t eventsel, instance, unitmask;
-	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
-
 	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 
-	if (lo_val == NULL || hi_val == NULL)
-		return;
-
 	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
-		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
+		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 				*lo_base_addr, *hi_base_addr);
-		return;
+		return -ENXIO;
+	}
+
+	if (lo_val && hi_val) {
+		uint32_t eventsel, instance, unitmask;
+		uint32_t instance_10, instance_5432, instance_76;
+
+		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
+		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
+		instance = DF_V3_6_GET_INSTANCE(config);
+
+		instance_10 = instance & 0x3;
+		instance_5432 = (instance >> 2) & 0xf;
+		instance_76 = (instance >> 6) & 0x3;
+
+		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
+		*hi_val = (instance_76 << 29) | instance_5432;
 	}
 
-	eventsel = GET_EVENT(config);
-	instance = GET_INSTANCE(config);
-	unitmask = GET_UNITMASK(config);
-
-	es_5_0 = eventsel & 0x3FUL;
-	es_13_6 = instance;
-	es_13_0 = (es_13_6 << 6) + es_5_0;
-	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
-	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
-	es_7_0 = es_13_0 & 0x0FFUL;
-	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
-	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
+	return 0;
 }
 
 /* assign df performance counters for read */
@@ -218,26 +300,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 				   uint64_t config,
 				   int *is_assigned)
 {
-
-	struct df_v3_6_event_mask *mask;
 	int i, target_cntr;
 
-	target_cntr = -1;
-
 	*is_assigned = 0;
 
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0) {
 		*is_assigned = 1;
 		return 0;
 	}
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if (mask->config_assign_mask[i] == 0ULL) {
-			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
+			adev->df_perfmon_config_assign_mask[i] =
+							config & 0x0FFFFFFUL;
 			return 0;
 		}
 	}
@@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 				     uint64_t config)
 {
-
-	struct df_v3_6_event_mask *mask;
-	int target_cntr;
-
-	target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
-
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0)
-		mask->config_assign_mask[target_cntr] = 0ULL;
-
+		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 }
 
-/*
- * get xgmi link counters via programmable data fabric (df) counters (max 4)
- * using cake tx event.
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @count -> counters to pass
- *
- */
-
-static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       uint64_t *count)
-{
-	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
-				      &hi_base_addr);
-
-	if ((lo_base_addr == 0) || (hi_base_addr == 0))
-		return;
-
-	lo_val = RREG32_PCIE(lo_base_addr);
-	hi_val = RREG32_PCIE(hi_base_addr);
 
-	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
-}
-
-/*
- * reset xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
-static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
-					 int instance)
+static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
+					 uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr;
-	uint64_t config;
-
-	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
 
 	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 				      &hi_base_addr);
@@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
 	WREG32_PCIE(hi_base_addr, 0UL);
 }
 
-/*
- * add xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
 
-static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
-				      int instance)
+static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
 	int ret, is_assigned;
 
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
 	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
 
 	if (ret || is_assigned)
 		return ret;
 
-	df_v3_6_pmc_get_ctrl_settings(adev,
+	ret = df_v3_6_pmc_get_ctrl_settings(adev,
 			config,
 			&lo_base_addr,
 			&hi_base_addr,
 			&lo_val,
 			&hi_val);
 
+	if (ret)
+		return ret;
+
+	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
 	WREG32_PCIE(lo_base_addr, lo_val);
 	WREG32_PCIE(hi_base_addr, hi_val);
 
 	return ret;
 }
 
-
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either resume or assign event via df perfmon
- *
- */
-
-static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
-					int instance,
-					int is_enable)
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+			     int is_enable)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-	int ret;
-
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	if (is_enable) {
+	int ret = 0;
 
-		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
-
-		if (ret)
-			return ret;
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
 
-	} else {
+		df_v3_6_reset_perfmon_cntr(adev, config);
 
-		config = GET_INSTANCE_CONFIG(instance);
+		if (is_enable) {
+			ret = df_v3_6_add_perfmon_cntr(adev, config);
+		} else {
+			ret = df_v3_6_pmc_get_ctrl_settings(adev,
+					config,
+					&lo_base_addr,
+					&hi_base_addr,
+					NULL,
+					NULL);
 
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
+			if (ret)
+				return ret;
 
-		if (lo_base_addr == 0)
-			return -EINVAL;
+			lo_val = RREG32_PCIE(lo_base_addr);
 
-		lo_val = RREG32_PCIE(lo_base_addr);
+			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+		}
 
-		ret = 0;
+		break;
+	default:
+		break;
 	}
 
 	return ret;
-
 }
 
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either pause or unassign event via df perfmon
- *
- */
-
-static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       int is_disable)
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+			    int is_disable)
 {
-
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	if (is_disable) {
-		df_v3_6_reset_xgmi_link_cntr(adev, instance);
-		df_v3_6_pmc_release_cntr(adev, config);
-	} else {
-
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
-
-		if ((lo_base_addr == 0) || (hi_base_addr == 0))
-			return -EINVAL;
-
-		lo_val = RREG32_PCIE(lo_base_addr);
-
-		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
-	}
-
-	return 0;
-}
-
-static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
-			     int is_enable)
-{
-	int xgmi_tx_link, ret = 0;
+	int ret = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-		if (xgmi_tx_link >= 0)
-			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
-						      is_enable);
+		ret = df_v3_6_pmc_get_ctrl_settings(adev,
+			config,
+			&lo_base_addr,
+			&hi_base_addr,
+			NULL,
+			NULL);
 
 		if (ret)
 			return ret;
 
-		ret = 0;
-		break;
-	default:
-		break;
-	}
+		lo_val = RREG32_PCIE(lo_base_addr);
 
-	return ret;
-}
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
-			    int is_disable)
-{
-	int xgmi_tx_link, ret = 0;
+		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
 
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-			if (xgmi_tx_link >= 0) {
-				ret = df_v3_6_stop_xgmi_link_cntr(adev,
-								  xgmi_tx_link,
-								  is_disable);
-				if (ret)
-					return ret;
-			}
-
-			ret = 0;
-			break;
+		if (is_disable)
+			df_v3_6_pmc_release_cntr(adev, config);
+
+		break;
 	default:
 		break;
 	}
@@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 				  uint64_t config,
 				  uint64_t *count)
 {
-
-	int xgmi_tx_link;
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	*count = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 
-		if (xgmi_tx_link >= 0) {
-			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
-			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
-		}
+		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+		if ((lo_base_addr == 0) || (hi_base_addr == 0))
+			return;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+		hi_val = RREG32_PCIE(hi_base_addr);
+
+		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+
+		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
+			*count = 0;
+
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 
 		break;
+
 	default:
 		break;
 	}
-
 }
 
 const struct amdgpu_df_funcs df_v3_6_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
index fcffd807764d..76998541bc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
 };
 
 /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
-#define AMDGPU_DF_MAX_COUNTERS		4
+#define DF_V3_6_MAX_COUNTERS		4
 
 /* get flags from df perfmon config */
-#define GET_EVENT(x)			(x & 0xFFUL)
-#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
-#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
-#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
-					| ((0x046ULL + x) << 8) \
-					| (0x02 << 16))
-
-/* df event conf macros */
-#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
-#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
+#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
+#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
+#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
+#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
 
+extern const struct attribute_group *df_v3_6_attr_groups[];
 extern const struct amdgpu_df_funcs df_v3_6_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 78bd4fc07bab..7fee24ea7863 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
 static int soc15_common_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
 
 	/* enable pcie gen2/3 link */
 	soc15_pcie_gen3_enable(adev);
@@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
 	 */
 	if (adev->nbio_funcs->remap_hdp_registers)
 		adev->nbio_funcs->remap_hdp_registers(adev);
+
+	adev->df_funcs->init(adev);
+
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 	/* HW doorbell routing policy: doorbell writing not
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* RE: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found] ` <20190620005313.56371-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-20  0:55   ` Kim, Jonathan
  2019-06-20  4:15   ` Kuehling, Felix
  2019-06-20 13:49   ` Alex Deucher
  2 siblings, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-20  0:55 UTC (permalink / raw)
  To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org,
	Kuehling, Felix

For Felix as requested - Simplifying adev df perfmon config assign mask to be fixed length and related simplification.

Jon

-----Original Message-----
From: Kim, Jonathan <Jonathan.Kim@amd.com> 
Sent: Wednesday, June 19, 2019 8:53 PM
To: amd-gfx@lists.freedesktop.org
Cc: Kim, Jonathan <Jonathan.Kim@amd.com>; Kim, Jonathan <Jonathan.Kim@amd.com>
Subject: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons

v4: fixed kzalloc error check and modified df func init to return error code

v3: fixed cleanup by adding fini to free up adev df config counters

v2: simplified by removing xgmi references in function names and moving to generic df function names.  fixed issue by removing hardcoded cake tx data events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return error code.

add pmu attribute groups and structures for perf events.
add sysfs to track available df perfmon counters fix overflow handling in perfmon counter reads.

Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +  drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------  drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
 4 files changed, 231 insertions(+), 258 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d355e9a09ad1..91cfcc7be5c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -732,6 +732,7 @@ struct amd_powerplay {  };
 
 #define AMDGPU_RESET_MAGIC_NUM 64
+#define AMDGPU_MAX_DF_PERFMONS 16
 struct amdgpu_device {
 	struct device			*dev;
 	struct drm_device		*ddev;
@@ -962,6 +963,7 @@ struct amdgpu_device {
 	long				compute_timeout;
 
 	uint64_t			unique_id;
+	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return  #endif
 
 #include "amdgpu_object.h"
+
+/* used by df_v3_6.c and amdgpu_pmu.c */
+#define AMDGPU_PMU_ATTR(_name, _object)				\
+static ssize_t								\
+_name##_show(struct device *dev,					\
+			       struct device_attribute *attr,		\
+			       char *page)				\
+{									\
+	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
+	return sprintf(page, _object "\n");				\
+}									\
+									\
+static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
+
 #endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index 8c09bf994acd..12e3e67013d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -30,8 +30,104 @@
 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
 				       16, 32, 0, 0, 0, 2, 4, 8};
 
+/* init df format attrs */
+AMDGPU_PMU_ATTR(event,		"config:0-7");
+AMDGPU_PMU_ATTR(instance,	"config:8-15");
+AMDGPU_PMU_ATTR(umask,		"config:16-23");
+
+/* df format attributes  */
+static struct attribute *df_v3_6_format_attrs[] = {
+	&pmu_attr_event.attr,
+	&pmu_attr_instance.attr,
+	&pmu_attr_umask.attr,
+	NULL
+};
+
+/* df format attribute group */
+static struct attribute_group df_v3_6_format_attr_group = {
+	.name = "format",
+	.attrs = df_v3_6_format_attrs,
+};
+
+/* df event attrs */
+AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
+		      "event=0x7,instance=0x46,umask=0x2");
+AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
+		      "event=0x7,instance=0x47,umask=0x2");
+AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
+		      "event=0x7,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
+		      "event=0x7,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
+		      "event=0xb,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x46,umask=0x8");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
+		      "event=0xb,instance=0x47,umask=0x8");
+
+/* df event attributes  */
+static struct attribute *df_v3_6_event_attrs[] = {
+	&pmu_attr_cake0_pcsout_txdata.attr,
+	&pmu_attr_cake1_pcsout_txdata.attr,
+	&pmu_attr_cake0_pcsout_txmeta.attr,
+	&pmu_attr_cake1_pcsout_txmeta.attr,
+	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
+	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
+	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
+	NULL
+};
+
+/* df event attribute group */
+static struct attribute_group df_v3_6_event_attr_group = {
+	.name = "events",
+	.attrs = df_v3_6_event_attrs
+};
+
+/* df event attr groups  */
+const struct attribute_group *df_v3_6_attr_groups[] = {
+		&df_v3_6_format_attr_group,
+		&df_v3_6_event_attr_group,
+		NULL
+};
+
+/* get the number of df counters available */ static ssize_t 
+df_v3_6_get_df_cntr_avail(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *ddev;
+	int i, count;
+
+	ddev = dev_get_drvdata(dev);
+	adev = ddev->dev_private;
+	count = 0;
+
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0)
+			count++;
+	}
+
+	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
+}
+
+/* device attr for available perfmon counters */ static 
+DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
+
+/* init perfmons */
 static void df_v3_6_init(struct amdgpu_device *adev)  {
+	int i, ret;
+
+	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
+	if (ret)
+		DRM_ERROR("failed to create file for available df counters\n");
+
+	for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
+		adev->df_perfmon_config_assign_mask[i] = 0;
 }
 
 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev, @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 		*flags |= AMD_CG_SUPPORT_DF_MGCG;
 }
 
-/* hold counter assignment per gpu struct */ -struct df_v3_6_event_mask {
-		struct amdgpu_device gpu;
-		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
-};
-
 /* get assigned df perfmon ctr as int */ -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
-				      uint64_t config,
-				      int *counter)
+static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
-	struct df_v3_6_event_mask *mask;
 	int i;
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
-			*counter = i;
-			return;
-		}
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if ((config & 0x0FFFFFFUL) ==
+					adev->df_perfmon_config_assign_mask[i])
+			return i;
 	}
+
+	return -EINVAL;
 }
 
 /* get address based on counter assignment */ @@ -136,10 +223,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 				 uint32_t *lo_base_addr,
 				 uint32_t *hi_base_addr)
 {
-
-	int target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr < 0)
 		return;
@@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,  }
 
 /* get control counter settings i.e. address and values to set */ -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 					  uint64_t config,
 					  uint32_t *lo_base_addr,
 					  uint32_t *hi_base_addr,
 					  uint32_t *lo_val,
 					  uint32_t *hi_val)
 {
-
-	uint32_t eventsel, instance, unitmask;
-	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
-
 	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 
-	if (lo_val == NULL || hi_val == NULL)
-		return;
-
 	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
-		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
+		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 				*lo_base_addr, *hi_base_addr);
-		return;
+		return -ENXIO;
+	}
+
+	if (lo_val && hi_val) {
+		uint32_t eventsel, instance, unitmask;
+		uint32_t instance_10, instance_5432, instance_76;
+
+		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
+		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
+		instance = DF_V3_6_GET_INSTANCE(config);
+
+		instance_10 = instance & 0x3;
+		instance_5432 = (instance >> 2) & 0xf;
+		instance_76 = (instance >> 6) & 0x3;
+
+		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
+		*hi_val = (instance_76 << 29) | instance_5432;
 	}
 
-	eventsel = GET_EVENT(config);
-	instance = GET_INSTANCE(config);
-	unitmask = GET_UNITMASK(config);
-
-	es_5_0 = eventsel & 0x3FUL;
-	es_13_6 = instance;
-	es_13_0 = (es_13_6 << 6) + es_5_0;
-	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
-	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
-	es_7_0 = es_13_0 & 0x0FFUL;
-	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
-	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
+	return 0;
 }
 
 /* assign df performance counters for read */ @@ -218,26 +300,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 				   uint64_t config,
 				   int *is_assigned)
 {
-
-	struct df_v3_6_event_mask *mask;
 	int i, target_cntr;
 
-	target_cntr = -1;
-
 	*is_assigned = 0;
 
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0) {
 		*is_assigned = 1;
 		return 0;
 	}
 
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
-
-	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
-		if (mask->config_assign_mask[i] == 0ULL) {
-			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
+	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
+			adev->df_perfmon_config_assign_mask[i] =
+							config & 0x0FFFFFFUL;
 			return 0;
 		}
 	}
@@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,  static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 				     uint64_t config)
 {
-
-	struct df_v3_6_event_mask *mask;
-	int target_cntr;
-
-	target_cntr = -1;
-
-	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
-
-	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 
 	if (target_cntr >= 0)
-		mask->config_assign_mask[target_cntr] = 0ULL;
-
+		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 }
 
-/*
- * get xgmi link counters via programmable data fabric (df) counters (max 4)
- * using cake tx event.
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @count -> counters to pass
- *
- */
-
-static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       uint64_t *count)
-{
-	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
-				      &hi_base_addr);
-
-	if ((lo_base_addr == 0) || (hi_base_addr == 0))
-		return;
-
-	lo_val = RREG32_PCIE(lo_base_addr);
-	hi_val = RREG32_PCIE(hi_base_addr);
 
-	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
-}
-
-/*
- * reset xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
-static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
-					 int instance)
+static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
+					 uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr;
-	uint64_t config;
-
-	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
 
 	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 				      &hi_base_addr);
@@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
 	WREG32_PCIE(hi_base_addr, 0UL);
 }
 
-/*
- * add xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- *
- */
 
-static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
-				      int instance)
+static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
+				      uint64_t config)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
-	uint64_t config;
 	int ret, is_assigned;
 
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
 	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
 
 	if (ret || is_assigned)
 		return ret;
 
-	df_v3_6_pmc_get_ctrl_settings(adev,
+	ret = df_v3_6_pmc_get_ctrl_settings(adev,
 			config,
 			&lo_base_addr,
 			&hi_base_addr,
 			&lo_val,
 			&hi_val);
 
+	if (ret)
+		return ret;
+
+	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
 	WREG32_PCIE(lo_base_addr, lo_val);
 	WREG32_PCIE(hi_base_addr, hi_val);
 
 	return ret;
 }
 
-
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either resume or assign event via df perfmon
- *
- */
-
-static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
-					int instance,
-					int is_enable)
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+			     int is_enable)
 {
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-	int ret;
-
-	if (instance < 0 || instance > 1)
-		return -EINVAL;
-
-	if (is_enable) {
+	int ret = 0;
 
-		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
-
-		if (ret)
-			return ret;
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
 
-	} else {
+		df_v3_6_reset_perfmon_cntr(adev, config);
 
-		config = GET_INSTANCE_CONFIG(instance);
+		if (is_enable) {
+			ret = df_v3_6_add_perfmon_cntr(adev, config);
+		} else {
+			ret = df_v3_6_pmc_get_ctrl_settings(adev,
+					config,
+					&lo_base_addr,
+					&hi_base_addr,
+					NULL,
+					NULL);
 
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
+			if (ret)
+				return ret;
 
-		if (lo_base_addr == 0)
-			return -EINVAL;
+			lo_val = RREG32_PCIE(lo_base_addr);
 
-		lo_val = RREG32_PCIE(lo_base_addr);
+			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+		}
 
-		ret = 0;
+		break;
+	default:
+		break;
 	}
 
 	return ret;
-
 }
 
-/*
- * start xgmi link counters
- *
- * @adev -> amdgpu device
- * @instance-> currently cake has 2 links to poll on vega20
- * @is_enable -> either pause or unassign event via df perfmon
- *
- */
-
-static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
-				       int instance,
-				       int is_disable)
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+			    int is_disable)
 {
-
 	uint32_t lo_base_addr, hi_base_addr, lo_val;
-	uint64_t config;
-
-	config = GET_INSTANCE_CONFIG(instance);
-
-	if (is_disable) {
-		df_v3_6_reset_xgmi_link_cntr(adev, instance);
-		df_v3_6_pmc_release_cntr(adev, config);
-	} else {
-
-		df_v3_6_pmc_get_ctrl_settings(adev,
-				config,
-				&lo_base_addr,
-				&hi_base_addr,
-				NULL,
-				NULL);
-
-		if ((lo_base_addr == 0) || (hi_base_addr == 0))
-			return -EINVAL;
-
-		lo_val = RREG32_PCIE(lo_base_addr);
-
-		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
-	}
-
-	return 0;
-}
-
-static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
-			     int is_enable)
-{
-	int xgmi_tx_link, ret = 0;
+	int ret = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-		if (xgmi_tx_link >= 0)
-			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
-						      is_enable);
+		ret = df_v3_6_pmc_get_ctrl_settings(adev,
+			config,
+			&lo_base_addr,
+			&hi_base_addr,
+			NULL,
+			NULL);
 
 		if (ret)
 			return ret;
 
-		ret = 0;
-		break;
-	default:
-		break;
-	}
+		lo_val = RREG32_PCIE(lo_base_addr);
 
-	return ret;
-}
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+				config, lo_base_addr, hi_base_addr, lo_val);
 
-static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
-			    int is_disable)
-{
-	int xgmi_tx_link, ret = 0;
+		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
 
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
-
-			if (xgmi_tx_link >= 0) {
-				ret = df_v3_6_stop_xgmi_link_cntr(adev,
-								  xgmi_tx_link,
-								  is_disable);
-				if (ret)
-					return ret;
-			}
-
-			ret = 0;
-			break;
+		if (is_disable)
+			df_v3_6_pmc_release_cntr(adev, config);
+
+		break;
 	default:
 		break;
 	}
@@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 				  uint64_t config,
 				  uint64_t *count)
 {
-
-	int xgmi_tx_link;
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	*count = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA20:
-		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
-					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 
-		if (xgmi_tx_link >= 0) {
-			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
-			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
-		}
+		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+		if ((lo_base_addr == 0) || (hi_base_addr == 0))
+			return;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+		hi_val = RREG32_PCIE(hi_base_addr);
+
+		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+
+		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
+			*count = 0;
+
+		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 
 		break;
+
 	default:
 		break;
 	}
-
 }
 
 const struct amdgpu_df_funcs df_v3_6_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
index fcffd807764d..76998541bc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
 };
 
 /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
-#define AMDGPU_DF_MAX_COUNTERS		4
+#define DF_V3_6_MAX_COUNTERS		4
 
 /* get flags from df perfmon config */
-#define GET_EVENT(x)			(x & 0xFFUL)
-#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
-#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
-#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
-					| ((0x046ULL + x) << 8) \
-					| (0x02 << 16))
-
-/* df event conf macros */
-#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
-#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
-		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
+#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
+#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
+#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
+#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
 
+extern const struct attribute_group *df_v3_6_attr_groups[];
 extern const struct amdgpu_df_funcs df_v3_6_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 78bd4fc07bab..7fee24ea7863 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)  static int soc15_common_hw_init(void *handle)  {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
 
 	/* enable pcie gen2/3 link */
 	soc15_pcie_gen3_enable(adev);
@@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
 	 */
 	if (adev->nbio_funcs->remap_hdp_registers)
 		adev->nbio_funcs->remap_hdp_registers(adev);
+
+	adev->df_funcs->init(adev);
+
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 	/* HW doorbell routing policy: doorbell writing not
--
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found] ` <20190620005313.56371-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-20  0:55   ` Kim, Jonathan
@ 2019-06-20  4:15   ` Kuehling, Felix
       [not found]     ` <84933584-80af-79ae-76ce-b2350e0aa69f-5C7GfCeVMHo@public.gmane.org>
  2019-06-20 13:49   ` Alex Deucher
  2 siblings, 1 reply; 15+ messages in thread
From: Kuehling, Felix @ 2019-06-20  4:15 UTC (permalink / raw)
  To: Kim, Jonathan,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

On 2019-06-19 20:53, Kim, Jonathan wrote:
> v4: fixed kzalloc error check and modified df func init to return error code

This comment isn't applicable any more because there is no more kzalloc. 
Maybe remove the review version history and instead update the 
description of what this commit changes.

Two more nit-picks inline. With that fixed, this patch is Reviewed-by: 
Felix Kuehling <Felix.Kuehling@amd.com>

>
> v3: fixed cleanup by adding fini to free up adev df config counters
>
> v2: simplified by removing xgmi references in function names and moving to
> generic df function names.  fixed issue by removing hardcoded cake tx data
> events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
> error code.
>
> add pmu attribute groups and structures for perf events.
> add sysfs to track available df perfmon counters
> fix overflow handling in perfmon counter reads.
>
> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------
>   drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>   drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
>   4 files changed, 231 insertions(+), 258 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index d355e9a09ad1..91cfcc7be5c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -732,6 +732,7 @@ struct amd_powerplay {
>   };
>   
>   #define AMDGPU_RESET_MAGIC_NUM 64
> +#define AMDGPU_MAX_DF_PERFMONS 16
>   struct amdgpu_device {
>   	struct device			*dev;
>   	struct drm_device		*ddev;
> @@ -962,6 +963,7 @@ struct amdgpu_device {
>   	long				compute_timeout;
>   
>   	uint64_t			unique_id;
> +	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
I was sort of expecting to see DF_V3_6_MAX_COUNTERS as the array size? 
Maybe using a more generic definition is not a bad thing to avoid tying 
this to a specific IP version. But I think 4 are enough for now. You 
definitely don't need 16 for any currently supported GPUs. This can be 
updated in the future if needed.


>   };
>   
>   static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
> @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>   #endif
>   
>   #include "amdgpu_object.h"
> +
> +/* used by df_v3_6.c and amdgpu_pmu.c */
> +#define AMDGPU_PMU_ATTR(_name, _object)				\
> +static ssize_t								\
> +_name##_show(struct device *dev,					\
> +			       struct device_attribute *attr,		\
> +			       char *page)				\
> +{									\
> +	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
> +	return sprintf(page, _object "\n");				\
> +}									\
> +									\
> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
> +
>   #endif
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> index 8c09bf994acd..12e3e67013d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> @@ -30,8 +30,104 @@
>   static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>   				       16, 32, 0, 0, 0, 2, 4, 8};
>   
> +/* init df format attrs */
> +AMDGPU_PMU_ATTR(event,		"config:0-7");
> +AMDGPU_PMU_ATTR(instance,	"config:8-15");
> +AMDGPU_PMU_ATTR(umask,		"config:16-23");
> +
> +/* df format attributes  */
> +static struct attribute *df_v3_6_format_attrs[] = {
> +	&pmu_attr_event.attr,
> +	&pmu_attr_instance.attr,
> +	&pmu_attr_umask.attr,
> +	NULL
> +};
> +
> +/* df format attribute group */
> +static struct attribute_group df_v3_6_format_attr_group = {
> +	.name = "format",
> +	.attrs = df_v3_6_format_attrs,
> +};
> +
> +/* df event attrs */
> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
> +		      "event=0x7,instance=0x46,umask=0x2");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
> +		      "event=0x7,instance=0x47,umask=0x2");
> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
> +		      "event=0x7,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
> +		      "event=0x7,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
> +		      "event=0xb,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x46,umask=0x8");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
> +		      "event=0xb,instance=0x47,umask=0x8");
> +
> +/* df event attributes  */
> +static struct attribute *df_v3_6_event_attrs[] = {
> +	&pmu_attr_cake0_pcsout_txdata.attr,
> +	&pmu_attr_cake1_pcsout_txdata.attr,
> +	&pmu_attr_cake0_pcsout_txmeta.attr,
> +	&pmu_attr_cake1_pcsout_txmeta.attr,
> +	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
> +	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
> +	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
> +	NULL
> +};
> +
> +/* df event attribute group */
> +static struct attribute_group df_v3_6_event_attr_group = {
> +	.name = "events",
> +	.attrs = df_v3_6_event_attrs
> +};
> +
> +/* df event attr groups  */
> +const struct attribute_group *df_v3_6_attr_groups[] = {
> +		&df_v3_6_format_attr_group,
> +		&df_v3_6_event_attr_group,
> +		NULL
> +};
> +
> +/* get the number of df counters available */
> +static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
> +		struct device_attribute *attr,
> +		char *buf)
> +{
> +	struct amdgpu_device *adev;
> +	struct drm_device *ddev;
> +	int i, count;
> +
> +	ddev = dev_get_drvdata(dev);
> +	adev = ddev->dev_private;
> +	count = 0;
> +
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0)
> +			count++;
> +	}
> +
> +	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
> +}
> +
> +/* device attr for available perfmon counters */
> +static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
> +
> +/* init perfmons */
>   static void df_v3_6_init(struct amdgpu_device *adev)
>   {
> +	int i, ret;
> +
> +	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
> +	if (ret)
> +		DRM_ERROR("failed to create file for available df counters\n");
> +
> +	for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
> +		adev->df_perfmon_config_assign_mask[i] = 0;
>   }
>   
>   static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
> @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>   		*flags |= AMD_CG_SUPPORT_DF_MGCG;
>   }
>   
> -/* hold counter assignment per gpu struct */
> -struct df_v3_6_event_mask {
> -		struct amdgpu_device gpu;
> -		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
> -};
> -
>   /* get assigned df perfmon ctr as int */
> -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> -				      uint64_t config,
> -				      int *counter)
> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
> -	struct df_v3_6_event_mask *mask;
>   	int i;
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
> -			*counter = i;
> -			return;
> -		}
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if ((config & 0x0FFFFFFUL) ==
> +					adev->df_perfmon_config_assign_mask[i])
> +			return i;
>   	}
> +
> +	return -EINVAL;
>   }
>   
>   /* get address based on counter assignment */
> @@ -136,10 +223,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>   				 uint32_t *lo_base_addr,
>   				 uint32_t *hi_base_addr)
>   {
> -
> -	int target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr < 0)
>   		return;
> @@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>   }
>   
>   /* get control counter settings i.e. address and values to set */
> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>   					  uint64_t config,
>   					  uint32_t *lo_base_addr,
>   					  uint32_t *hi_base_addr,
>   					  uint32_t *lo_val,
>   					  uint32_t *hi_val)
>   {
> -
> -	uint32_t eventsel, instance, unitmask;
> -	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
> -
>   	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
>   
> -	if (lo_val == NULL || hi_val == NULL)
> -		return;
> -
>   	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
> -		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
> +		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>   				*lo_base_addr, *hi_base_addr);
> -		return;
> +		return -ENXIO;
> +	}
> +
> +	if (lo_val && hi_val) {
> +		uint32_t eventsel, instance, unitmask;
> +		uint32_t instance_10, instance_5432, instance_76;
> +
> +		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
> +		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
> +		instance = DF_V3_6_GET_INSTANCE(config);
> +
> +		instance_10 = instance & 0x3;
> +		instance_5432 = (instance >> 2) & 0xf;
> +		instance_76 = (instance >> 6) & 0x3;
> +
> +		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
> +		*hi_val = (instance_76 << 29) | instance_5432;
>   	}
>   
> -	eventsel = GET_EVENT(config);
> -	instance = GET_INSTANCE(config);
> -	unitmask = GET_UNITMASK(config);
> -
> -	es_5_0 = eventsel & 0x3FUL;
> -	es_13_6 = instance;
> -	es_13_0 = (es_13_6 << 6) + es_5_0;
> -	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
> -	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
> -	es_7_0 = es_13_0 & 0x0FFUL;
> -	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
> -	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
> +	return 0;
>   }
>   
>   /* assign df performance counters for read */
> @@ -218,26 +300,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   				   uint64_t config,
>   				   int *is_assigned)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
>   	int i, target_cntr;
>   
> -	target_cntr = -1;
> -
>   	*is_assigned = 0;
>   
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0) {
>   		*is_assigned = 1;
>   		return 0;
>   	}
>   
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -		if (mask->config_assign_mask[i] == 0ULL) {
> -			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
> +			adev->df_perfmon_config_assign_mask[i] =
> +							config & 0x0FFFFFFUL;
>   			return 0;
>   		}
>   	}
> @@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>   static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>   				     uint64_t config)
>   {
> -
> -	struct df_v3_6_event_mask *mask;
> -	int target_cntr;
> -
> -	target_cntr = -1;
> -
> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> -
> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>   
>   	if (target_cntr >= 0)
> -		mask->config_assign_mask[target_cntr] = 0ULL;
> -
> +		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>   }
>   
> -/*
> - * get xgmi link counters via programmable data fabric (df) counters (max 4)
> - * using cake tx event.
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @count -> counters to pass
> - *
> - */
> -
> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       uint64_t *count)
> -{
> -	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> -				      &hi_base_addr);
> -
> -	if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -		return;
> -
> -	lo_val = RREG32_PCIE(lo_base_addr);
> -	hi_val = RREG32_PCIE(hi_base_addr);
>   
> -	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> -}
> -
> -/*
> - * reset xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
> -					 int instance)
> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
> +					 uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr;
> -	uint64_t config;
> -
> -	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>   
>   	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>   				      &hi_base_addr);
> @@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>   	WREG32_PCIE(hi_base_addr, 0UL);
>   }
>   
> -/*
> - * add xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
>   
> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
> -				      int instance)
> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
> +				      uint64_t config)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -	uint64_t config;
>   	int ret, is_assigned;
>   
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
>   	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>   
>   	if (ret || is_assigned)
>   		return ret;
>   
> -	df_v3_6_pmc_get_ctrl_settings(adev,
> +	ret = df_v3_6_pmc_get_ctrl_settings(adev,
>   			config,
>   			&lo_base_addr,
>   			&hi_base_addr,
>   			&lo_val,
>   			&hi_val);
>   
> +	if (ret)
> +		return ret;
> +
> +	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
> +
>   	WREG32_PCIE(lo_base_addr, lo_val);
>   	WREG32_PCIE(hi_base_addr, hi_val);
>   
>   	return ret;
>   }
>   
> -
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either resume or assign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
> -					int instance,
> -					int is_enable)
> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> +			     int is_enable)
>   {
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -	int ret;
> -
> -	if (instance < 0 || instance > 1)
> -		return -EINVAL;
> -
> -	if (is_enable) {
> +	int ret = 0;
>   
> -		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
> -
> -		if (ret)
> -			return ret;
> +	switch (adev->asic_type) {
> +	case CHIP_VEGA20:
>   
> -	} else {
> +		df_v3_6_reset_perfmon_cntr(adev, config);
>   
> -		config = GET_INSTANCE_CONFIG(instance);
> +		if (is_enable) {
> +			ret = df_v3_6_add_perfmon_cntr(adev, config);
> +		} else {
> +			ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +					config,
> +					&lo_base_addr,
> +					&hi_base_addr,
> +					NULL,
> +					NULL);
>   
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> +			if (ret)
> +				return ret;
>   
> -		if (lo_base_addr == 0)
> -			return -EINVAL;
> +			lo_val = RREG32_PCIE(lo_base_addr);
>   
> -		lo_val = RREG32_PCIE(lo_base_addr);
> +			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +		}
>   
> -		ret = 0;
> +		break;
> +	default:
> +		break;
>   	}
>   
>   	return ret;
> -
>   }
>   
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either pause or unassign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
> -				       int instance,
> -				       int is_disable)
> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> +			    int is_disable)
>   {
> -
>   	uint32_t lo_base_addr, hi_base_addr, lo_val;
> -	uint64_t config;
> -
> -	config = GET_INSTANCE_CONFIG(instance);
> -
> -	if (is_disable) {
> -		df_v3_6_reset_xgmi_link_cntr(adev, instance);
> -		df_v3_6_pmc_release_cntr(adev, config);
> -	} else {
> -
> -		df_v3_6_pmc_get_ctrl_settings(adev,
> -				config,
> -				&lo_base_addr,
> -				&hi_base_addr,
> -				NULL,
> -				NULL);
> -
> -		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -			return -EINVAL;
> -
> -		lo_val = RREG32_PCIE(lo_base_addr);
> -
> -		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
> -	}
> -
> -	return 0;
> -}
> -
> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> -			     int is_enable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +	int ret = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -		if (xgmi_tx_link >= 0)
> -			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
> -						      is_enable);
> +		ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +			config,
> +			&lo_base_addr,
> +			&hi_base_addr,
> +			NULL,
> +			NULL);
>   
>   		if (ret)
>   			return ret;
>   
> -		ret = 0;
> -		break;
> -	default:
> -		break;
> -	}
> +		lo_val = RREG32_PCIE(lo_base_addr);
>   
> -	return ret;
> -}
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +				config, lo_base_addr, hi_base_addr, lo_val);
>   
> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> -			    int is_disable)
> -{
> -	int xgmi_tx_link, ret = 0;
> +		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>   
> -	switch (adev->asic_type) {
> -	case CHIP_VEGA20:
> -			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -			if (xgmi_tx_link >= 0) {
> -				ret = df_v3_6_stop_xgmi_link_cntr(adev,
> -								  xgmi_tx_link,
> -								  is_disable);
> -				if (ret)
> -					return ret;
> -			}
> -
> -			ret = 0;
> -			break;
> +		if (is_disable)
> +			df_v3_6_pmc_release_cntr(adev, config);
> +
> +		break;
>   	default:
>   		break;
>   	}
> @@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>   				  uint64_t config,
>   				  uint64_t *count)
>   {
> -
> -	int xgmi_tx_link;
> +	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> +	*count = 0;
>   
>   	switch (adev->asic_type) {
>   	case CHIP_VEGA20:
> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>   
> -		if (xgmi_tx_link >= 0) {
> -			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
> -			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
> -		}
> +		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> +				      &hi_base_addr);
> +
> +		if ((lo_base_addr == 0) || (hi_base_addr == 0))
> +			return;
> +
> +		lo_val = RREG32_PCIE(lo_base_addr);
> +		hi_val = RREG32_PCIE(hi_base_addr);
> +
> +		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> +
> +		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
> +			*count = 0;
> +
> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>   
>   		break;
> +
>   	default:
>   		break;
>   	}
> -
>   }
>   
>   const struct amdgpu_df_funcs df_v3_6_funcs = {
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> index fcffd807764d..76998541bc30 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>   };
>   
>   /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
> -#define AMDGPU_DF_MAX_COUNTERS		4
> +#define DF_V3_6_MAX_COUNTERS		4
>   
>   /* get flags from df perfmon config */
> -#define GET_EVENT(x)			(x & 0xFFUL)
> -#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
> -#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
> -#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
> -					| ((0x046ULL + x) << 8) \
> -					| (0x02 << 16))
> -
> -/* df event conf macros */
> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
> -		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
> +#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
> +#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
> +#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
> +#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
>   
> +extern const struct attribute_group *df_v3_6_attr_groups[];
>   extern const struct amdgpu_df_funcs df_v3_6_funcs;
>   
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 78bd4fc07bab..7fee24ea7863 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>   static int soc15_common_hw_init(void *handle)
>   {
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +	int ret;

ret is not needed any more.

Regards,
   Felix


>   
>   	/* enable pcie gen2/3 link */
>   	soc15_pcie_gen3_enable(adev);
> @@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
>   	 */
>   	if (adev->nbio_funcs->remap_hdp_registers)
>   		adev->nbio_funcs->remap_hdp_registers(adev);
> +
> +	adev->df_funcs->init(adev);
> +
>   	/* enable the doorbell aperture */
>   	soc15_enable_doorbell_aperture(adev, true);
>   	/* HW doorbell routing policy: doorbell writing not
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found]     ` <84933584-80af-79ae-76ce-b2350e0aa69f-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-20  4:17       ` Kuehling, Felix
       [not found]         ` <5a8a03fe-af1a-370f-b4c5-5784cfcdc22c-5C7GfCeVMHo@public.gmane.org>
  0 siblings, 1 reply; 15+ messages in thread
From: Kuehling, Felix @ 2019-06-20  4:17 UTC (permalink / raw)
  To: Kim, Jonathan,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

Um, it looks like you already submitted the change before this review.

In case you're not aware, amd-staging-drm-next is set up to auto-submit 
with out the gerrit code review and pre-submission test we have on 
amd-kfd-staging. Do not push to amd-staging-drm-next unless you really 
mean to submit your changes. Please apply the two fixes I pointed out 
below in a separate patch.

Thanks,
   Felix

On 2019-06-20 0:15, Kuehling, Felix wrote:
> On 2019-06-19 20:53, Kim, Jonathan wrote:
>> v4: fixed kzalloc error check and modified df func init to return error code
> This comment isn't applicable any more because there is no more kzalloc.
> Maybe remove the review version history and instead update the
> description of what this commit changes.
>
> Two more nit-picks inline. With that fixed, this patch is Reviewed-by:
> Felix Kuehling <Felix.Kuehling@amd.com>
>
>> v3: fixed cleanup by adding fini to free up adev df config counters
>>
>> v2: simplified by removing xgmi references in function names and moving to
>> generic df function names.  fixed issue by removing hardcoded cake tx data
>> events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
>> error code.
>>
>> add pmu attribute groups and structures for perf events.
>> add sysfs to track available df perfmon counters
>> fix overflow handling in perfmon counter reads.
>>
>> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
>> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
>> ---
>>    drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +
>>    drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------
>>    drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>>    drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
>>    4 files changed, 231 insertions(+), 258 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> index d355e9a09ad1..91cfcc7be5c1 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> @@ -732,6 +732,7 @@ struct amd_powerplay {
>>    };
>>    
>>    #define AMDGPU_RESET_MAGIC_NUM 64
>> +#define AMDGPU_MAX_DF_PERFMONS 16
>>    struct amdgpu_device {
>>    	struct device			*dev;
>>    	struct drm_device		*ddev;
>> @@ -962,6 +963,7 @@ struct amdgpu_device {
>>    	long				compute_timeout;
>>    
>>    	uint64_t			unique_id;
>> +	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
> I was sort of expecting to see DF_V3_6_MAX_COUNTERS as the array size?
> Maybe using a more generic definition is not a bad thing to avoid tying
> this to a specific IP version. But I think 4 are enough for now. You
> definitely don't need 16 for any currently supported GPUs. This can be
> updated in the future if needed.
>
>
>>    };
>>    
>>    static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
>> @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>>    #endif
>>    
>>    #include "amdgpu_object.h"
>> +
>> +/* used by df_v3_6.c and amdgpu_pmu.c */
>> +#define AMDGPU_PMU_ATTR(_name, _object)				\
>> +static ssize_t								\
>> +_name##_show(struct device *dev,					\
>> +			       struct device_attribute *attr,		\
>> +			       char *page)				\
>> +{									\
>> +	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
>> +	return sprintf(page, _object "\n");				\
>> +}									\
>> +									\
>> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
>> +
>>    #endif
>> +
>> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> index 8c09bf994acd..12e3e67013d9 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> @@ -30,8 +30,104 @@
>>    static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>>    				       16, 32, 0, 0, 0, 2, 4, 8};
>>    
>> +/* init df format attrs */
>> +AMDGPU_PMU_ATTR(event,		"config:0-7");
>> +AMDGPU_PMU_ATTR(instance,	"config:8-15");
>> +AMDGPU_PMU_ATTR(umask,		"config:16-23");
>> +
>> +/* df format attributes  */
>> +static struct attribute *df_v3_6_format_attrs[] = {
>> +	&pmu_attr_event.attr,
>> +	&pmu_attr_instance.attr,
>> +	&pmu_attr_umask.attr,
>> +	NULL
>> +};
>> +
>> +/* df format attribute group */
>> +static struct attribute_group df_v3_6_format_attr_group = {
>> +	.name = "format",
>> +	.attrs = df_v3_6_format_attrs,
>> +};
>> +
>> +/* df event attrs */
>> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
>> +		      "event=0x7,instance=0x46,umask=0x2");
>> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
>> +		      "event=0x7,instance=0x47,umask=0x2");
>> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
>> +		      "event=0x7,instance=0x46,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
>> +		      "event=0x7,instance=0x47,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
>> +		      "event=0xb,instance=0x46,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
>> +		      "event=0xb,instance=0x47,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
>> +		      "event=0xb,instance=0x46,umask=0x8");
>> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
>> +		      "event=0xb,instance=0x47,umask=0x8");
>> +
>> +/* df event attributes  */
>> +static struct attribute *df_v3_6_event_attrs[] = {
>> +	&pmu_attr_cake0_pcsout_txdata.attr,
>> +	&pmu_attr_cake1_pcsout_txdata.attr,
>> +	&pmu_attr_cake0_pcsout_txmeta.attr,
>> +	&pmu_attr_cake1_pcsout_txmeta.attr,
>> +	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
>> +	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
>> +	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
>> +	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
>> +	NULL
>> +};
>> +
>> +/* df event attribute group */
>> +static struct attribute_group df_v3_6_event_attr_group = {
>> +	.name = "events",
>> +	.attrs = df_v3_6_event_attrs
>> +};
>> +
>> +/* df event attr groups  */
>> +const struct attribute_group *df_v3_6_attr_groups[] = {
>> +		&df_v3_6_format_attr_group,
>> +		&df_v3_6_event_attr_group,
>> +		NULL
>> +};
>> +
>> +/* get the number of df counters available */
>> +static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
>> +		struct device_attribute *attr,
>> +		char *buf)
>> +{
>> +	struct amdgpu_device *adev;
>> +	struct drm_device *ddev;
>> +	int i, count;
>> +
>> +	ddev = dev_get_drvdata(dev);
>> +	adev = ddev->dev_private;
>> +	count = 0;
>> +
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if (adev->df_perfmon_config_assign_mask[i] == 0)
>> +			count++;
>> +	}
>> +
>> +	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
>> +}
>> +
>> +/* device attr for available perfmon counters */
>> +static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
>> +
>> +/* init perfmons */
>>    static void df_v3_6_init(struct amdgpu_device *adev)
>>    {
>> +	int i, ret;
>> +
>> +	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
>> +	if (ret)
>> +		DRM_ERROR("failed to create file for available df counters\n");
>> +
>> +	for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
>> +		adev->df_perfmon_config_assign_mask[i] = 0;
>>    }
>>    
>>    static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
>> @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>>    		*flags |= AMD_CG_SUPPORT_DF_MGCG;
>>    }
>>    
>> -/* hold counter assignment per gpu struct */
>> -struct df_v3_6_event_mask {
>> -		struct amdgpu_device gpu;
>> -		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
>> -};
>> -
>>    /* get assigned df perfmon ctr as int */
>> -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
>> -				      uint64_t config,
>> -				      int *counter)
>> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
>> +				      uint64_t config)
>>    {
>> -	struct df_v3_6_event_mask *mask;
>>    	int i;
>>    
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> -
>> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
>> -		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
>> -			*counter = i;
>> -			return;
>> -		}
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if ((config & 0x0FFFFFFUL) ==
>> +					adev->df_perfmon_config_assign_mask[i])
>> +			return i;
>>    	}
>> +
>> +	return -EINVAL;
>>    }
>>    
>>    /* get address based on counter assignment */
>> @@ -136,10 +223,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>>    				 uint32_t *lo_base_addr,
>>    				 uint32_t *hi_base_addr)
>>    {
>> -
>> -	int target_cntr = -1;
>> -
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr < 0)
>>    		return;
>> @@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>>    }
>>    
>>    /* get control counter settings i.e. address and values to set */
>> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>>    					  uint64_t config,
>>    					  uint32_t *lo_base_addr,
>>    					  uint32_t *hi_base_addr,
>>    					  uint32_t *lo_val,
>>    					  uint32_t *hi_val)
>>    {
>> -
>> -	uint32_t eventsel, instance, unitmask;
>> -	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
>> -
>>    	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
>>    
>> -	if (lo_val == NULL || hi_val == NULL)
>> -		return;
>> -
>>    	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
>> -		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
>> +		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>>    				*lo_base_addr, *hi_base_addr);
>> -		return;
>> +		return -ENXIO;
>> +	}
>> +
>> +	if (lo_val && hi_val) {
>> +		uint32_t eventsel, instance, unitmask;
>> +		uint32_t instance_10, instance_5432, instance_76;
>> +
>> +		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
>> +		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
>> +		instance = DF_V3_6_GET_INSTANCE(config);
>> +
>> +		instance_10 = instance & 0x3;
>> +		instance_5432 = (instance >> 2) & 0xf;
>> +		instance_76 = (instance >> 6) & 0x3;
>> +
>> +		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
>> +		*hi_val = (instance_76 << 29) | instance_5432;
>>    	}
>>    
>> -	eventsel = GET_EVENT(config);
>> -	instance = GET_INSTANCE(config);
>> -	unitmask = GET_UNITMASK(config);
>> -
>> -	es_5_0 = eventsel & 0x3FUL;
>> -	es_13_6 = instance;
>> -	es_13_0 = (es_13_6 << 6) + es_5_0;
>> -	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
>> -	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
>> -	es_7_0 = es_13_0 & 0x0FFUL;
>> -	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
>> -	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
>> +	return 0;
>>    }
>>    
>>    /* assign df performance counters for read */
>> @@ -218,26 +300,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>>    				   uint64_t config,
>>    				   int *is_assigned)
>>    {
>> -
>> -	struct df_v3_6_event_mask *mask;
>>    	int i, target_cntr;
>>    
>> -	target_cntr = -1;
>> -
>>    	*is_assigned = 0;
>>    
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> +	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr >= 0) {
>>    		*is_assigned = 1;
>>    		return 0;
>>    	}
>>    
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> -
>> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
>> -		if (mask->config_assign_mask[i] == 0ULL) {
>> -			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
>> +			adev->df_perfmon_config_assign_mask[i] =
>> +							config & 0x0FFFFFFUL;
>>    			return 0;
>>    		}
>>    	}
>> @@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>>    static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>>    				     uint64_t config)
>>    {
>> -
>> -	struct df_v3_6_event_mask *mask;
>> -	int target_cntr;
>> -
>> -	target_cntr = -1;
>> -
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> -
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr >= 0)
>> -		mask->config_assign_mask[target_cntr] = 0ULL;
>> -
>> +		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>>    }
>>    
>> -/*
>> - * get xgmi link counters via programmable data fabric (df) counters (max 4)
>> - * using cake tx event.
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @count -> counters to pass
>> - *
>> - */
>> -
>> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				       int instance,
>> -				       uint64_t *count)
>> -{
>> -	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> -	uint64_t config;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>> -	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>> -				      &hi_base_addr);
>> -
>> -	if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> -		return;
>> -
>> -	lo_val = RREG32_PCIE(lo_base_addr);
>> -	hi_val = RREG32_PCIE(hi_base_addr);
>>    
>> -	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
>> -}
>> -
>> -/*
>> - * reset xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - *
>> - */
>> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>> -					 int instance)
>> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
>> +					 uint64_t config)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr;
>> -	uint64_t config;
>> -
>> -	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>>    
>>    	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>>    				      &hi_base_addr);
>> @@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>>    	WREG32_PCIE(hi_base_addr, 0UL);
>>    }
>>    
>> -/*
>> - * add xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - *
>> - */
>>    
>> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				      int instance)
>> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
>> +				      uint64_t config)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> -	uint64_t config;
>>    	int ret, is_assigned;
>>    
>> -	if (instance < 0 || instance > 1)
>> -		return -EINVAL;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>>    	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>>    
>>    	if (ret || is_assigned)
>>    		return ret;
>>    
>> -	df_v3_6_pmc_get_ctrl_settings(adev,
>> +	ret = df_v3_6_pmc_get_ctrl_settings(adev,
>>    			config,
>>    			&lo_base_addr,
>>    			&hi_base_addr,
>>    			&lo_val,
>>    			&hi_val);
>>    
>> +	if (ret)
>> +		return ret;
>> +
>> +	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
>> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>> +
>>    	WREG32_PCIE(lo_base_addr, lo_val);
>>    	WREG32_PCIE(hi_base_addr, hi_val);
>>    
>>    	return ret;
>>    }
>>    
>> -
>> -/*
>> - * start xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @is_enable -> either resume or assign event via df perfmon
>> - *
>> - */
>> -
>> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
>> -					int instance,
>> -					int is_enable)
>> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
>> +			     int is_enable)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val;
>> -	uint64_t config;
>> -	int ret;
>> -
>> -	if (instance < 0 || instance > 1)
>> -		return -EINVAL;
>> -
>> -	if (is_enable) {
>> +	int ret = 0;
>>    
>> -		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
>> -
>> -		if (ret)
>> -			return ret;
>> +	switch (adev->asic_type) {
>> +	case CHIP_VEGA20:
>>    
>> -	} else {
>> +		df_v3_6_reset_perfmon_cntr(adev, config);
>>    
>> -		config = GET_INSTANCE_CONFIG(instance);
>> +		if (is_enable) {
>> +			ret = df_v3_6_add_perfmon_cntr(adev, config);
>> +		} else {
>> +			ret = df_v3_6_pmc_get_ctrl_settings(adev,
>> +					config,
>> +					&lo_base_addr,
>> +					&hi_base_addr,
>> +					NULL,
>> +					NULL);
>>    
>> -		df_v3_6_pmc_get_ctrl_settings(adev,
>> -				config,
>> -				&lo_base_addr,
>> -				&hi_base_addr,
>> -				NULL,
>> -				NULL);
>> +			if (ret)
>> +				return ret;
>>    
>> -		if (lo_base_addr == 0)
>> -			return -EINVAL;
>> +			lo_val = RREG32_PCIE(lo_base_addr);
>>    
>> -		lo_val = RREG32_PCIE(lo_base_addr);
>> +			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
>> +				config, lo_base_addr, hi_base_addr, lo_val);
>>    
>> -		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
>> +			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
>> +		}
>>    
>> -		ret = 0;
>> +		break;
>> +	default:
>> +		break;
>>    	}
>>    
>>    	return ret;
>> -
>>    }
>>    
>> -/*
>> - * start xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @is_enable -> either pause or unassign event via df perfmon
>> - *
>> - */
>> -
>> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				       int instance,
>> -				       int is_disable)
>> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
>> +			    int is_disable)
>>    {
>> -
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val;
>> -	uint64_t config;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>> -	if (is_disable) {
>> -		df_v3_6_reset_xgmi_link_cntr(adev, instance);
>> -		df_v3_6_pmc_release_cntr(adev, config);
>> -	} else {
>> -
>> -		df_v3_6_pmc_get_ctrl_settings(adev,
>> -				config,
>> -				&lo_base_addr,
>> -				&hi_base_addr,
>> -				NULL,
>> -				NULL);
>> -
>> -		if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> -			return -EINVAL;
>> -
>> -		lo_val = RREG32_PCIE(lo_base_addr);
>> -
>> -		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>> -	}
>> -
>> -	return 0;
>> -}
>> -
>> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
>> -			     int is_enable)
>> -{
>> -	int xgmi_tx_link, ret = 0;
>> +	int ret = 0;
>>    
>>    	switch (adev->asic_type) {
>>    	case CHIP_VEGA20:
>> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>> -
>> -		if (xgmi_tx_link >= 0)
>> -			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
>> -						      is_enable);
>> +		ret = df_v3_6_pmc_get_ctrl_settings(adev,
>> +			config,
>> +			&lo_base_addr,
>> +			&hi_base_addr,
>> +			NULL,
>> +			NULL);
>>    
>>    		if (ret)
>>    			return ret;
>>    
>> -		ret = 0;
>> -		break;
>> -	default:
>> -		break;
>> -	}
>> +		lo_val = RREG32_PCIE(lo_base_addr);
>>    
>> -	return ret;
>> -}
>> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
>> +				config, lo_base_addr, hi_base_addr, lo_val);
>>    
>> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
>> -			    int is_disable)
>> -{
>> -	int xgmi_tx_link, ret = 0;
>> +		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>>    
>> -	switch (adev->asic_type) {
>> -	case CHIP_VEGA20:
>> -			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>> -
>> -			if (xgmi_tx_link >= 0) {
>> -				ret = df_v3_6_stop_xgmi_link_cntr(adev,
>> -								  xgmi_tx_link,
>> -								  is_disable);
>> -				if (ret)
>> -					return ret;
>> -			}
>> -
>> -			ret = 0;
>> -			break;
>> +		if (is_disable)
>> +			df_v3_6_pmc_release_cntr(adev, config);
>> +
>> +		break;
>>    	default:
>>    		break;
>>    	}
>> @@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>>    				  uint64_t config,
>>    				  uint64_t *count)
>>    {
>> -
>> -	int xgmi_tx_link;
>> +	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> +	*count = 0;
>>    
>>    	switch (adev->asic_type) {
>>    	case CHIP_VEGA20:
>> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>>    
>> -		if (xgmi_tx_link >= 0) {
>> -			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
>> -			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
>> -		}
>> +		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>> +				      &hi_base_addr);
>> +
>> +		if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> +			return;
>> +
>> +		lo_val = RREG32_PCIE(lo_base_addr);
>> +		hi_val = RREG32_PCIE(hi_base_addr);
>> +
>> +		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
>> +
>> +		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
>> +			*count = 0;
>> +
>> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
>> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>>    
>>    		break;
>> +
>>    	default:
>>    		break;
>>    	}
>> -
>>    }
>>    
>>    const struct amdgpu_df_funcs df_v3_6_funcs = {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> index fcffd807764d..76998541bc30 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>>    };
>>    
>>    /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
>> -#define AMDGPU_DF_MAX_COUNTERS		4
>> +#define DF_V3_6_MAX_COUNTERS		4
>>    
>>    /* get flags from df perfmon config */
>> -#define GET_EVENT(x)			(x & 0xFFUL)
>> -#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
>> -#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
>> -#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
>> -					| ((0x046ULL + x) << 8) \
>> -					| (0x02 << 16))
>> -
>> -/* df event conf macros */
>> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
>> -		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
>> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
>> -		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
>> +#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
>> +#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
>> +#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
>> +#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
>>    
>> +extern const struct attribute_group *df_v3_6_attr_groups[];
>>    extern const struct amdgpu_df_funcs df_v3_6_funcs;
>>    
>>    #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> index 78bd4fc07bab..7fee24ea7863 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>>    static int soc15_common_hw_init(void *handle)
>>    {
>>    	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>> +	int ret;
> ret is not needed any more.
>
> Regards,
>     Felix
>
>
>>    
>>    	/* enable pcie gen2/3 link */
>>    	soc15_pcie_gen3_enable(adev);
>> @@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
>>    	 */
>>    	if (adev->nbio_funcs->remap_hdp_registers)
>>    		adev->nbio_funcs->remap_hdp_registers(adev);
>> +
>> +	adev->df_funcs->init(adev);
>> +
>>    	/* enable the doorbell aperture */
>>    	soc15_enable_doorbell_aperture(adev, true);
>>    	/* HW doorbell routing policy: doorbell writing not
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* RE: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found]         ` <5a8a03fe-af1a-370f-b4c5-5784cfcdc22c-5C7GfCeVMHo@public.gmane.org>
@ 2019-06-20  5:21           ` Kim, Jonathan
  0 siblings, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-20  5:21 UTC (permalink / raw)
  To: Kuehling, Felix,
	amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

Apologies.  I confused the review approval from the 2nd patch in the series.  Re-submitted requested changes for review in new patch - "drm/amdgpu: set max df perfmon to 4".

Thanks,

Jon

-----Original Message-----
From: Kuehling, Felix <Felix.Kuehling@amd.com> 
Sent: Thursday, June 20, 2019 12:18 AM
To: Kim, Jonathan <Jonathan.Kim@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons

Um, it looks like you already submitted the change before this review.

In case you're not aware, amd-staging-drm-next is set up to auto-submit with out the gerrit code review and pre-submission test we have on amd-kfd-staging. Do not push to amd-staging-drm-next unless you really mean to submit your changes. Please apply the two fixes I pointed out below in a separate patch.

Thanks,
   Felix

On 2019-06-20 0:15, Kuehling, Felix wrote:
> On 2019-06-19 20:53, Kim, Jonathan wrote:
>> v4: fixed kzalloc error check and modified df func init to return 
>> error code
> This comment isn't applicable any more because there is no more kzalloc.
> Maybe remove the review version history and instead update the 
> description of what this commit changes.
>
> Two more nit-picks inline. With that fixed, this patch is Reviewed-by:
> Felix Kuehling <Felix.Kuehling@amd.com>
>
>> v3: fixed cleanup by adding fini to free up adev df config counters
>>
>> v2: simplified by removing xgmi references in function names and 
>> moving to generic df function names.  fixed issue by removing 
>> hardcoded cake tx data events. streamlined error handling by having  
>> df_v3_6_pmc_get_ctrl return error code.
>>
>> add pmu attribute groups and structures for perf events.
>> add sysfs to track available df perfmon counters fix overflow 
>> handling in perfmon counter reads.
>>
>> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
>> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
>> ---
>>    drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +
>>    drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------
>>    drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>>    drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
>>    4 files changed, 231 insertions(+), 258 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> index d355e9a09ad1..91cfcc7be5c1 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> @@ -732,6 +732,7 @@ struct amd_powerplay {
>>    };
>>    
>>    #define AMDGPU_RESET_MAGIC_NUM 64
>> +#define AMDGPU_MAX_DF_PERFMONS 16
>>    struct amdgpu_device {
>>    	struct device			*dev;
>>    	struct drm_device		*ddev;
>> @@ -962,6 +963,7 @@ struct amdgpu_device {
>>    	long				compute_timeout;
>>    
>>    	uint64_t			unique_id;
>> +	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
> I was sort of expecting to see DF_V3_6_MAX_COUNTERS as the array size?
> Maybe using a more generic definition is not a bad thing to avoid 
> tying this to a specific IP version. But I think 4 are enough for now. 
> You definitely don't need 16 for any currently supported GPUs. This 
> can be updated in the future if needed.
>
>
>>    };
>>    
>>    static inline struct amdgpu_device *amdgpu_ttm_adev(struct 
>> ttm_bo_device *bdev) @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>>    #endif
>>    
>>    #include "amdgpu_object.h"
>> +
>> +/* used by df_v3_6.c and amdgpu_pmu.c */
>> +#define AMDGPU_PMU_ATTR(_name, _object)				\
>> +static ssize_t								\
>> +_name##_show(struct device *dev,					\
>> +			       struct device_attribute *attr,		\
>> +			       char *page)				\
>> +{									\
>> +	BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);			\
>> +	return sprintf(page, _object "\n");				\
>> +}									\
>> +									\
>> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
>> +
>>    #endif
>> +
>> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c 
>> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> index 8c09bf994acd..12e3e67013d9 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
>> @@ -30,8 +30,104 @@
>>    static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>>    				       16, 32, 0, 0, 0, 2, 4, 8};
>>    
>> +/* init df format attrs */
>> +AMDGPU_PMU_ATTR(event,		"config:0-7");
>> +AMDGPU_PMU_ATTR(instance,	"config:8-15");
>> +AMDGPU_PMU_ATTR(umask,		"config:16-23");
>> +
>> +/* df format attributes  */
>> +static struct attribute *df_v3_6_format_attrs[] = {
>> +	&pmu_attr_event.attr,
>> +	&pmu_attr_instance.attr,
>> +	&pmu_attr_umask.attr,
>> +	NULL
>> +};
>> +
>> +/* df format attribute group */
>> +static struct attribute_group df_v3_6_format_attr_group = {
>> +	.name = "format",
>> +	.attrs = df_v3_6_format_attrs,
>> +};
>> +
>> +/* df event attrs */
>> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
>> +		      "event=0x7,instance=0x46,umask=0x2");
>> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
>> +		      "event=0x7,instance=0x47,umask=0x2");
>> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
>> +		      "event=0x7,instance=0x46,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
>> +		      "event=0x7,instance=0x47,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
>> +		      "event=0xb,instance=0x46,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
>> +		      "event=0xb,instance=0x47,umask=0x4");
>> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
>> +		      "event=0xb,instance=0x46,umask=0x8");
>> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
>> +		      "event=0xb,instance=0x47,umask=0x8");
>> +
>> +/* df event attributes  */
>> +static struct attribute *df_v3_6_event_attrs[] = {
>> +	&pmu_attr_cake0_pcsout_txdata.attr,
>> +	&pmu_attr_cake1_pcsout_txdata.attr,
>> +	&pmu_attr_cake0_pcsout_txmeta.attr,
>> +	&pmu_attr_cake1_pcsout_txmeta.attr,
>> +	&pmu_attr_cake0_ftiinstat_reqalloc.attr,
>> +	&pmu_attr_cake1_ftiinstat_reqalloc.attr,
>> +	&pmu_attr_cake0_ftiinstat_rspalloc.attr,
>> +	&pmu_attr_cake1_ftiinstat_rspalloc.attr,
>> +	NULL
>> +};
>> +
>> +/* df event attribute group */
>> +static struct attribute_group df_v3_6_event_attr_group = {
>> +	.name = "events",
>> +	.attrs = df_v3_6_event_attrs
>> +};
>> +
>> +/* df event attr groups  */
>> +const struct attribute_group *df_v3_6_attr_groups[] = {
>> +		&df_v3_6_format_attr_group,
>> +		&df_v3_6_event_attr_group,
>> +		NULL
>> +};
>> +
>> +/* get the number of df counters available */ static ssize_t 
>> +df_v3_6_get_df_cntr_avail(struct device *dev,
>> +		struct device_attribute *attr,
>> +		char *buf)
>> +{
>> +	struct amdgpu_device *adev;
>> +	struct drm_device *ddev;
>> +	int i, count;
>> +
>> +	ddev = dev_get_drvdata(dev);
>> +	adev = ddev->dev_private;
>> +	count = 0;
>> +
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if (adev->df_perfmon_config_assign_mask[i] == 0)
>> +			count++;
>> +	}
>> +
>> +	return snprintf(buf, PAGE_SIZE,	"%i\n", count);
>> +}
>> +
>> +/* device attr for available perfmon counters */ static 
>> +DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, 
>> +NULL);
>> +
>> +/* init perfmons */
>>    static void df_v3_6_init(struct amdgpu_device *adev)
>>    {
>> +	int i, ret;
>> +
>> +	ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
>> +	if (ret)
>> +		DRM_ERROR("failed to create file for available df counters\n");
>> +
>> +	for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
>> +		adev->df_perfmon_config_assign_mask[i] = 0;
>>    }
>>    
>>    static void df_v3_6_enable_broadcast_mode(struct amdgpu_device 
>> *adev, @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>>    		*flags |= AMD_CG_SUPPORT_DF_MGCG;
>>    }
>>    
>> -/* hold counter assignment per gpu struct */ -struct 
>> df_v3_6_event_mask {
>> -		struct amdgpu_device gpu;
>> -		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
>> -};
>> -
>>    /* get assigned df perfmon ctr as int */ -static void 
>> df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
>> -				      uint64_t config,
>> -				      int *counter)
>> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
>> +				      uint64_t config)
>>    {
>> -	struct df_v3_6_event_mask *mask;
>>    	int i;
>>    
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> -
>> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
>> -		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
>> -			*counter = i;
>> -			return;
>> -		}
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if ((config & 0x0FFFFFFUL) ==
>> +					adev->df_perfmon_config_assign_mask[i])
>> +			return i;
>>    	}
>> +
>> +	return -EINVAL;
>>    }
>>    
>>    /* get address based on counter assignment */ @@ -136,10 +223,7 @@ 
>> static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>>    				 uint32_t *lo_base_addr,
>>    				 uint32_t *hi_base_addr)
>>    {
>> -
>> -	int target_cntr = -1;
>> -
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr < 0)
>>    		return;
>> @@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>>    }
>>    
>>    /* get control counter settings i.e. address and values to set */ 
>> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device 
>> *adev,
>> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>>    					  uint64_t config,
>>    					  uint32_t *lo_base_addr,
>>    					  uint32_t *hi_base_addr,
>>    					  uint32_t *lo_val,
>>    					  uint32_t *hi_val)
>>    {
>> -
>> -	uint32_t eventsel, instance, unitmask;
>> -	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
>> -
>>    	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, 
>> hi_base_addr);
>>    
>> -	if (lo_val == NULL || hi_val == NULL)
>> -		return;
>> -
>>    	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
>> -		DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
>> +		DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>>    				*lo_base_addr, *hi_base_addr);
>> -		return;
>> +		return -ENXIO;
>> +	}
>> +
>> +	if (lo_val && hi_val) {
>> +		uint32_t eventsel, instance, unitmask;
>> +		uint32_t instance_10, instance_5432, instance_76;
>> +
>> +		eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
>> +		unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
>> +		instance = DF_V3_6_GET_INSTANCE(config);
>> +
>> +		instance_10 = instance & 0x3;
>> +		instance_5432 = (instance >> 2) & 0xf;
>> +		instance_76 = (instance >> 6) & 0x3;
>> +
>> +		*lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
>> +		*hi_val = (instance_76 << 29) | instance_5432;
>>    	}
>>    
>> -	eventsel = GET_EVENT(config);
>> -	instance = GET_INSTANCE(config);
>> -	unitmask = GET_UNITMASK(config);
>> -
>> -	es_5_0 = eventsel & 0x3FUL;
>> -	es_13_6 = instance;
>> -	es_13_0 = (es_13_6 << 6) + es_5_0;
>> -	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
>> -	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
>> -	es_7_0 = es_13_0 & 0x0FFUL;
>> -	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
>> -	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
>> +	return 0;
>>    }
>>    
>>    /* assign df performance counters for read */ @@ -218,26 +300,21 
>> @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>>    				   uint64_t config,
>>    				   int *is_assigned)
>>    {
>> -
>> -	struct df_v3_6_event_mask *mask;
>>    	int i, target_cntr;
>>    
>> -	target_cntr = -1;
>> -
>>    	*is_assigned = 0;
>>    
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> +	target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr >= 0) {
>>    		*is_assigned = 1;
>>    		return 0;
>>    	}
>>    
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> -
>> -	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
>> -		if (mask->config_assign_mask[i] == 0ULL) {
>> -			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
>> +	for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
>> +		if (adev->df_perfmon_config_assign_mask[i] == 0U) {
>> +			adev->df_perfmon_config_assign_mask[i] =
>> +							config & 0x0FFFFFFUL;
>>    			return 0;
>>    		}
>>    	}
>> @@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>>    static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>>    				     uint64_t config)
>>    {
>> -
>> -	struct df_v3_6_event_mask *mask;
>> -	int target_cntr;
>> -
>> -	target_cntr = -1;
>> -
>> -	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
>> -
>> -	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
>> +	int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>>    
>>    	if (target_cntr >= 0)
>> -		mask->config_assign_mask[target_cntr] = 0ULL;
>> -
>> +		adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>>    }
>>    
>> -/*
>> - * get xgmi link counters via programmable data fabric (df) counters 
>> (max 4)
>> - * using cake tx event.
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @count -> counters to pass
>> - *
>> - */
>> -
>> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				       int instance,
>> -				       uint64_t *count)
>> -{
>> -	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> -	uint64_t config;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>> -	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>> -				      &hi_base_addr);
>> -
>> -	if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> -		return;
>> -
>> -	lo_val = RREG32_PCIE(lo_base_addr);
>> -	hi_val = RREG32_PCIE(hi_base_addr);
>>    
>> -	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
>> -}
>> -
>> -/*
>> - * reset xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - *
>> - */
>> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>> -					 int instance)
>> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
>> +					 uint64_t config)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr;
>> -	uint64_t config;
>> -
>> -	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>>    
>>    	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>>    				      &hi_base_addr);
>> @@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>>    	WREG32_PCIE(hi_base_addr, 0UL);
>>    }
>>    
>> -/*
>> - * add xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - *
>> - */
>>    
>> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				      int instance)
>> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
>> +				      uint64_t config)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> -	uint64_t config;
>>    	int ret, is_assigned;
>>    
>> -	if (instance < 0 || instance > 1)
>> -		return -EINVAL;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>>    	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>>    
>>    	if (ret || is_assigned)
>>    		return ret;
>>    
>> -	df_v3_6_pmc_get_ctrl_settings(adev,
>> +	ret = df_v3_6_pmc_get_ctrl_settings(adev,
>>    			config,
>>    			&lo_base_addr,
>>    			&hi_base_addr,
>>    			&lo_val,
>>    			&hi_val);
>>    
>> +	if (ret)
>> +		return ret;
>> +
>> +	DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
>> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>> +
>>    	WREG32_PCIE(lo_base_addr, lo_val);
>>    	WREG32_PCIE(hi_base_addr, hi_val);
>>    
>>    	return ret;
>>    }
>>    
>> -
>> -/*
>> - * start xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @is_enable -> either resume or assign event via df perfmon
>> - *
>> - */
>> -
>> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
>> -					int instance,
>> -					int is_enable)
>> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
>> +			     int is_enable)
>>    {
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val;
>> -	uint64_t config;
>> -	int ret;
>> -
>> -	if (instance < 0 || instance > 1)
>> -		return -EINVAL;
>> -
>> -	if (is_enable) {
>> +	int ret = 0;
>>    
>> -		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
>> -
>> -		if (ret)
>> -			return ret;
>> +	switch (adev->asic_type) {
>> +	case CHIP_VEGA20:
>>    
>> -	} else {
>> +		df_v3_6_reset_perfmon_cntr(adev, config);
>>    
>> -		config = GET_INSTANCE_CONFIG(instance);
>> +		if (is_enable) {
>> +			ret = df_v3_6_add_perfmon_cntr(adev, config);
>> +		} else {
>> +			ret = df_v3_6_pmc_get_ctrl_settings(adev,
>> +					config,
>> +					&lo_base_addr,
>> +					&hi_base_addr,
>> +					NULL,
>> +					NULL);
>>    
>> -		df_v3_6_pmc_get_ctrl_settings(adev,
>> -				config,
>> -				&lo_base_addr,
>> -				&hi_base_addr,
>> -				NULL,
>> -				NULL);
>> +			if (ret)
>> +				return ret;
>>    
>> -		if (lo_base_addr == 0)
>> -			return -EINVAL;
>> +			lo_val = RREG32_PCIE(lo_base_addr);
>>    
>> -		lo_val = RREG32_PCIE(lo_base_addr);
>> +			DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
>> +				config, lo_base_addr, hi_base_addr, lo_val);
>>    
>> -		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
>> +			WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
>> +		}
>>    
>> -		ret = 0;
>> +		break;
>> +	default:
>> +		break;
>>    	}
>>    
>>    	return ret;
>> -
>>    }
>>    
>> -/*
>> - * start xgmi link counters
>> - *
>> - * @adev -> amdgpu device
>> - * @instance-> currently cake has 2 links to poll on vega20
>> - * @is_enable -> either pause or unassign event via df perfmon
>> - *
>> - */
>> -
>> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
>> -				       int instance,
>> -				       int is_disable)
>> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
>> +			    int is_disable)
>>    {
>> -
>>    	uint32_t lo_base_addr, hi_base_addr, lo_val;
>> -	uint64_t config;
>> -
>> -	config = GET_INSTANCE_CONFIG(instance);
>> -
>> -	if (is_disable) {
>> -		df_v3_6_reset_xgmi_link_cntr(adev, instance);
>> -		df_v3_6_pmc_release_cntr(adev, config);
>> -	} else {
>> -
>> -		df_v3_6_pmc_get_ctrl_settings(adev,
>> -				config,
>> -				&lo_base_addr,
>> -				&hi_base_addr,
>> -				NULL,
>> -				NULL);
>> -
>> -		if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> -			return -EINVAL;
>> -
>> -		lo_val = RREG32_PCIE(lo_base_addr);
>> -
>> -		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>> -	}
>> -
>> -	return 0;
>> -}
>> -
>> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
>> -			     int is_enable)
>> -{
>> -	int xgmi_tx_link, ret = 0;
>> +	int ret = 0;
>>    
>>    	switch (adev->asic_type) {
>>    	case CHIP_VEGA20:
>> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>> -
>> -		if (xgmi_tx_link >= 0)
>> -			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
>> -						      is_enable);
>> +		ret = df_v3_6_pmc_get_ctrl_settings(adev,
>> +			config,
>> +			&lo_base_addr,
>> +			&hi_base_addr,
>> +			NULL,
>> +			NULL);
>>    
>>    		if (ret)
>>    			return ret;
>>    
>> -		ret = 0;
>> -		break;
>> -	default:
>> -		break;
>> -	}
>> +		lo_val = RREG32_PCIE(lo_base_addr);
>>    
>> -	return ret;
>> -}
>> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
>> +				config, lo_base_addr, hi_base_addr, lo_val);
>>    
>> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
>> -			    int is_disable)
>> -{
>> -	int xgmi_tx_link, ret = 0;
>> +		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>>    
>> -	switch (adev->asic_type) {
>> -	case CHIP_VEGA20:
>> -			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>> -
>> -			if (xgmi_tx_link >= 0) {
>> -				ret = df_v3_6_stop_xgmi_link_cntr(adev,
>> -								  xgmi_tx_link,
>> -								  is_disable);
>> -				if (ret)
>> -					return ret;
>> -			}
>> -
>> -			ret = 0;
>> -			break;
>> +		if (is_disable)
>> +			df_v3_6_pmc_release_cntr(adev, config);
>> +
>> +		break;
>>    	default:
>>    		break;
>>    	}
>> @@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>>    				  uint64_t config,
>>    				  uint64_t *count)
>>    {
>> -
>> -	int xgmi_tx_link;
>> +	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
>> +	*count = 0;
>>    
>>    	switch (adev->asic_type) {
>>    	case CHIP_VEGA20:
>> -		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
>> -					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>>    
>> -		if (xgmi_tx_link >= 0) {
>> -			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
>> -			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
>> -		}
>> +		df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>> +				      &hi_base_addr);
>> +
>> +		if ((lo_base_addr == 0) || (hi_base_addr == 0))
>> +			return;
>> +
>> +		lo_val = RREG32_PCIE(lo_base_addr);
>> +		hi_val = RREG32_PCIE(hi_base_addr);
>> +
>> +		*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
>> +
>> +		if (*count >= DF_V3_6_PERFMON_OVERFLOW)
>> +			*count = 0;
>> +
>> +		DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
>> +			config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>>    
>>    		break;
>> +
>>    	default:
>>    		break;
>>    	}
>> -
>>    }
>>    
>>    const struct amdgpu_df_funcs df_v3_6_funcs = { diff --git 
>> a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h 
>> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> index fcffd807764d..76998541bc30 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
>> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>>    };
>>    
>>    /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
>> -#define AMDGPU_DF_MAX_COUNTERS		4
>> +#define DF_V3_6_MAX_COUNTERS		4
>>    
>>    /* get flags from df perfmon config */
>> -#define GET_EVENT(x)			(x & 0xFFUL)
>> -#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
>> -#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
>> -#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
>> -					| ((0x046ULL + x) << 8) \
>> -					| (0x02 << 16))
>> -
>> -/* df event conf macros */
>> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
>> -		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
>> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
>> -		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
>> +#define DF_V3_6_GET_EVENT(x)		(x & 0xFFUL)
>> +#define DF_V3_6_GET_INSTANCE(x)		((x >> 8) & 0xFFUL)
>> +#define DF_V3_6_GET_UNITMASK(x)		((x >> 16) & 0xFFUL)
>> +#define DF_V3_6_PERFMON_OVERFLOW	0xFFFFFFFFFFFFULL
>>    
>> +extern const struct attribute_group *df_v3_6_attr_groups[];
>>    extern const struct amdgpu_df_funcs df_v3_6_funcs;
>>    
>>    #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c 
>> b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> index 78bd4fc07bab..7fee24ea7863 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>>    static int soc15_common_hw_init(void *handle)
>>    {
>>    	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>> +	int ret;
> ret is not needed any more.
>
> Regards,
>     Felix
>
>
>>    
>>    	/* enable pcie gen2/3 link */
>>    	soc15_pcie_gen3_enable(adev);
>> @@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
>>    	 */
>>    	if (adev->nbio_funcs->remap_hdp_registers)
>>    		adev->nbio_funcs->remap_hdp_registers(adev);
>> +
>> +	adev->df_funcs->init(adev);
>> +
>>    	/* enable the doorbell aperture */
>>    	soc15_enable_doorbell_aperture(adev, true);
>>    	/* HW doorbell routing policy: doorbell writing not
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found] ` <20190620005313.56371-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
  2019-06-20  0:55   ` Kim, Jonathan
  2019-06-20  4:15   ` Kuehling, Felix
@ 2019-06-20 13:49   ` Alex Deucher
       [not found]     ` <CADnq5_O81Mmf=4N+EL8P6ARrbhAo8wXRVKZB78N5Dj7e6+upLw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
  2 siblings, 1 reply; 15+ messages in thread
From: Alex Deucher @ 2019-06-20 13:49 UTC (permalink / raw)
  To: Kim, Jonathan; +Cc: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

On Wed, Jun 19, 2019 at 8:53 PM Kim, Jonathan <Jonathan.Kim@amd.com> wrote:
>
> v4: fixed kzalloc error check and modified df func init to return error code
>
> v3: fixed cleanup by adding fini to free up adev df config counters
>
> v2: simplified by removing xgmi references in function names and moving to
> generic df function names.  fixed issue by removing hardcoded cake tx data
> events. streamlined error handling by having  df_v3_6_pmc_get_ctrl return
> error code.
>
> add pmu attribute groups and structures for perf events.
> add sysfs to track available df perfmon counters
> fix overflow handling in perfmon counter reads.
>
> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +
>  drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------
>  drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>  drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
>  4 files changed, 231 insertions(+), 258 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index d355e9a09ad1..91cfcc7be5c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -732,6 +732,7 @@ struct amd_powerplay {
>  };
>
>  #define AMDGPU_RESET_MAGIC_NUM 64
> +#define AMDGPU_MAX_DF_PERFMONS 16
>  struct amdgpu_device {
>         struct device                   *dev;
>         struct drm_device               *ddev;
> @@ -962,6 +963,7 @@ struct amdgpu_device {
>         long                            compute_timeout;
>
>         uint64_t                        unique_id;
> +       uint64_t        df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
>  };
>
>  static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
> @@ -1201,4 +1203,19 @@ static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return
>  #endif
>
>  #include "amdgpu_object.h"
> +
> +/* used by df_v3_6.c and amdgpu_pmu.c */
> +#define AMDGPU_PMU_ATTR(_name, _object)                                \
> +static ssize_t                                                         \
> +_name##_show(struct device *dev,                                       \
> +                              struct device_attribute *attr,           \
> +                              char *page)                              \
> +{                                                                      \
> +       BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);                 \
> +       return sprintf(page, _object "\n");                             \
> +}                                                                      \
> +                                                                       \
> +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
> +
>  #endif
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> index 8c09bf994acd..12e3e67013d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> @@ -30,8 +30,104 @@
>  static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>                                        16, 32, 0, 0, 0, 2, 4, 8};
>
> +/* init df format attrs */
> +AMDGPU_PMU_ATTR(event,         "config:0-7");
> +AMDGPU_PMU_ATTR(instance,      "config:8-15");
> +AMDGPU_PMU_ATTR(umask,         "config:16-23");
> +
> +/* df format attributes  */
> +static struct attribute *df_v3_6_format_attrs[] = {
> +       &pmu_attr_event.attr,
> +       &pmu_attr_instance.attr,
> +       &pmu_attr_umask.attr,
> +       NULL
> +};
> +
> +/* df format attribute group */
> +static struct attribute_group df_v3_6_format_attr_group = {
> +       .name = "format",
> +       .attrs = df_v3_6_format_attrs,
> +};
> +
> +/* df event attrs */
> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
> +                     "event=0x7,instance=0x46,umask=0x2");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
> +                     "event=0x7,instance=0x47,umask=0x2");
> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
> +                     "event=0x7,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
> +                     "event=0x7,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
> +                     "event=0xb,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
> +                     "event=0xb,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
> +                     "event=0xb,instance=0x46,umask=0x8");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
> +                     "event=0xb,instance=0x47,umask=0x8");
> +
> +/* df event attributes  */
> +static struct attribute *df_v3_6_event_attrs[] = {
> +       &pmu_attr_cake0_pcsout_txdata.attr,
> +       &pmu_attr_cake1_pcsout_txdata.attr,
> +       &pmu_attr_cake0_pcsout_txmeta.attr,
> +       &pmu_attr_cake1_pcsout_txmeta.attr,
> +       &pmu_attr_cake0_ftiinstat_reqalloc.attr,
> +       &pmu_attr_cake1_ftiinstat_reqalloc.attr,
> +       &pmu_attr_cake0_ftiinstat_rspalloc.attr,
> +       &pmu_attr_cake1_ftiinstat_rspalloc.attr,
> +       NULL
> +};
> +
> +/* df event attribute group */
> +static struct attribute_group df_v3_6_event_attr_group = {
> +       .name = "events",
> +       .attrs = df_v3_6_event_attrs
> +};
> +
> +/* df event attr groups  */
> +const struct attribute_group *df_v3_6_attr_groups[] = {
> +               &df_v3_6_format_attr_group,
> +               &df_v3_6_event_attr_group,
> +               NULL
> +};
> +
> +/* get the number of df counters available */
> +static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
> +               struct device_attribute *attr,
> +               char *buf)
> +{
> +       struct amdgpu_device *adev;
> +       struct drm_device *ddev;
> +       int i, count;
> +
> +       ddev = dev_get_drvdata(dev);
> +       adev = ddev->dev_private;
> +       count = 0;
> +
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if (adev->df_perfmon_config_assign_mask[i] == 0)
> +                       count++;
> +       }
> +
> +       return snprintf(buf, PAGE_SIZE, "%i\n", count);
> +}
> +
> +/* device attr for available perfmon counters */
> +static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
> +
> +/* init perfmons */
>  static void df_v3_6_init(struct amdgpu_device *adev)
>  {
> +       int i, ret;
> +
> +       ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
> +       if (ret)
> +               DRM_ERROR("failed to create file for available df counters\n");
> +
> +       for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
> +               adev->df_perfmon_config_assign_mask[i] = 0;
>  }

You never addressed my comments about df init from before.  df_init
gets called from hw_init, so you shouldn't create files.  Move df init
to being called from soc15 sw_init/fini and probably rename the df
callbacks to sw_init and sw_fini so we know not to add hw state to
those functions.  As is now, we'll end up needlessly creating and
destorying files on suspend/resume.

Alex


>
>  static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
> @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>                 *flags |= AMD_CG_SUPPORT_DF_MGCG;
>  }
>
> -/* hold counter assignment per gpu struct */
> -struct df_v3_6_event_mask {
> -               struct amdgpu_device gpu;
> -               uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
> -};
> -
>  /* get assigned df perfmon ctr as int */
> -static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> -                                     uint64_t config,
> -                                     int *counter)
> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> +                                     uint64_t config)
>  {
> -       struct df_v3_6_event_mask *mask;
>         int i;
>
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -       for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -               if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
> -                       *counter = i;
> -                       return;
> -               }
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if ((config & 0x0FFFFFFUL) ==
> +                                       adev->df_perfmon_config_assign_mask[i])
> +                       return i;
>         }
> +
> +       return -EINVAL;
>  }
>
>  /* get address based on counter assignment */
> @@ -136,10 +223,7 @@ static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>                                  uint32_t *lo_base_addr,
>                                  uint32_t *hi_base_addr)
>  {
> -
> -       int target_cntr = -1;
> -
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +       int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr < 0)
>                 return;
> @@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
>  }
>
>  /* get control counter settings i.e. address and values to set */
> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>                                           uint64_t config,
>                                           uint32_t *lo_base_addr,
>                                           uint32_t *hi_base_addr,
>                                           uint32_t *lo_val,
>                                           uint32_t *hi_val)
>  {
> -
> -       uint32_t eventsel, instance, unitmask;
> -       uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
> -
>         df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
>
> -       if (lo_val == NULL || hi_val == NULL)
> -               return;
> -
>         if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
> -               DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
> +               DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
>                                 *lo_base_addr, *hi_base_addr);
> -               return;
> +               return -ENXIO;
> +       }
> +
> +       if (lo_val && hi_val) {
> +               uint32_t eventsel, instance, unitmask;
> +               uint32_t instance_10, instance_5432, instance_76;
> +
> +               eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
> +               unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
> +               instance = DF_V3_6_GET_INSTANCE(config);
> +
> +               instance_10 = instance & 0x3;
> +               instance_5432 = (instance >> 2) & 0xf;
> +               instance_76 = (instance >> 6) & 0x3;
> +
> +               *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
> +               *hi_val = (instance_76 << 29) | instance_5432;
>         }
>
> -       eventsel = GET_EVENT(config);
> -       instance = GET_INSTANCE(config);
> -       unitmask = GET_UNITMASK(config);
> -
> -       es_5_0 = eventsel & 0x3FUL;
> -       es_13_6 = instance;
> -       es_13_0 = (es_13_6 << 6) + es_5_0;
> -       es_13_12 = (es_13_0 & 0x03000UL) >> 12;
> -       es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
> -       es_7_0 = es_13_0 & 0x0FFUL;
> -       *lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
> -       *hi_val = (es_11_8 | ((es_13_12)<<(29)));
> +       return 0;
>  }
>
>  /* assign df performance counters for read */
> @@ -218,26 +300,21 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>                                    uint64_t config,
>                                    int *is_assigned)
>  {
> -
> -       struct df_v3_6_event_mask *mask;
>         int i, target_cntr;
>
> -       target_cntr = -1;
> -
>         *is_assigned = 0;
>
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +       target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr >= 0) {
>                 *is_assigned = 1;
>                 return 0;
>         }
>
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -       for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -               if (mask->config_assign_mask[i] == 0ULL) {
> -                       mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if (adev->df_perfmon_config_assign_mask[i] == 0U) {
> +                       adev->df_perfmon_config_assign_mask[i] =
> +                                                       config & 0x0FFFFFFUL;
>                         return 0;
>                 }
>         }
> @@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>  static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>                                      uint64_t config)
>  {
> -
> -       struct df_v3_6_event_mask *mask;
> -       int target_cntr;
> -
> -       target_cntr = -1;
> -
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> -
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> +       int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr >= 0)
> -               mask->config_assign_mask[target_cntr] = 0ULL;
> -
> +               adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
>  }
>
> -/*
> - * get xgmi link counters via programmable data fabric (df) counters (max 4)
> - * using cake tx event.
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @count -> counters to pass
> - *
> - */
> -
> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                      int instance,
> -                                      uint64_t *count)
> -{
> -       uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -       uint64_t config;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
> -       df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> -                                     &hi_base_addr);
> -
> -       if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -               return;
> -
> -       lo_val = RREG32_PCIE(lo_base_addr);
> -       hi_val = RREG32_PCIE(hi_base_addr);
>
> -       *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> -}
> -
> -/*
> - * reset xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                        int instance)
> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
> +                                        uint64_t config)
>  {
>         uint32_t lo_base_addr, hi_base_addr;
> -       uint64_t config;
> -
> -       config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>
>         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>                                       &hi_base_addr);
> @@ -320,185 +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>         WREG32_PCIE(hi_base_addr, 0UL);
>  }
>
> -/*
> - * add xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
>
> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                     int instance)
> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
> +                                     uint64_t config)
>  {
>         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -       uint64_t config;
>         int ret, is_assigned;
>
> -       if (instance < 0 || instance > 1)
> -               return -EINVAL;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
>         ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>
>         if (ret || is_assigned)
>                 return ret;
>
> -       df_v3_6_pmc_get_ctrl_settings(adev,
> +       ret = df_v3_6_pmc_get_ctrl_settings(adev,
>                         config,
>                         &lo_base_addr,
>                         &hi_base_addr,
>                         &lo_val,
>                         &hi_val);
>
> +       if (ret)
> +               return ret;
> +
> +       DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +                       config, lo_base_addr, hi_base_addr, lo_val, hi_val);
> +
>         WREG32_PCIE(lo_base_addr, lo_val);
>         WREG32_PCIE(hi_base_addr, hi_val);
>
>         return ret;
>  }
>
> -
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either resume or assign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                       int instance,
> -                                       int is_enable)
> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> +                            int is_enable)
>  {
>         uint32_t lo_base_addr, hi_base_addr, lo_val;
> -       uint64_t config;
> -       int ret;
> -
> -       if (instance < 0 || instance > 1)
> -               return -EINVAL;
> -
> -       if (is_enable) {
> +       int ret = 0;
>
> -               ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
> -
> -               if (ret)
> -                       return ret;
> +       switch (adev->asic_type) {
> +       case CHIP_VEGA20:
>
> -       } else {
> +               df_v3_6_reset_perfmon_cntr(adev, config);
>
> -               config = GET_INSTANCE_CONFIG(instance);
> +               if (is_enable) {
> +                       ret = df_v3_6_add_perfmon_cntr(adev, config);
> +               } else {
> +                       ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +                                       config,
> +                                       &lo_base_addr,
> +                                       &hi_base_addr,
> +                                       NULL,
> +                                       NULL);
>
> -               df_v3_6_pmc_get_ctrl_settings(adev,
> -                               config,
> -                               &lo_base_addr,
> -                               &hi_base_addr,
> -                               NULL,
> -                               NULL);
> +                       if (ret)
> +                               return ret;
>
> -               if (lo_base_addr == 0)
> -                       return -EINVAL;
> +                       lo_val = RREG32_PCIE(lo_base_addr);
>
> -               lo_val = RREG32_PCIE(lo_base_addr);
> +                       DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +                               config, lo_base_addr, hi_base_addr, lo_val);
>
> -               WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +                       WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +               }
>
> -               ret = 0;
> +               break;
> +       default:
> +               break;
>         }
>
>         return ret;
> -
>  }
>
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either pause or unassign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                      int instance,
> -                                      int is_disable)
> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> +                           int is_disable)
>  {
> -
>         uint32_t lo_base_addr, hi_base_addr, lo_val;
> -       uint64_t config;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
> -       if (is_disable) {
> -               df_v3_6_reset_xgmi_link_cntr(adev, instance);
> -               df_v3_6_pmc_release_cntr(adev, config);
> -       } else {
> -
> -               df_v3_6_pmc_get_ctrl_settings(adev,
> -                               config,
> -                               &lo_base_addr,
> -                               &hi_base_addr,
> -                               NULL,
> -                               NULL);
> -
> -               if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -                       return -EINVAL;
> -
> -               lo_val = RREG32_PCIE(lo_base_addr);
> -
> -               WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
> -       }
> -
> -       return 0;
> -}
> -
> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> -                            int is_enable)
> -{
> -       int xgmi_tx_link, ret = 0;
> +       int ret = 0;
>
>         switch (adev->asic_type) {
>         case CHIP_VEGA20:
> -               xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                                       : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -               if (xgmi_tx_link >= 0)
> -                       ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
> -                                                     is_enable);
> +               ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +                       config,
> +                       &lo_base_addr,
> +                       &hi_base_addr,
> +                       NULL,
> +                       NULL);
>
>                 if (ret)
>                         return ret;
>
> -               ret = 0;
> -               break;
> -       default:
> -               break;
> -       }
> +               lo_val = RREG32_PCIE(lo_base_addr);
>
> -       return ret;
> -}
> +               DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +                               config, lo_base_addr, hi_base_addr, lo_val);
>
> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> -                           int is_disable)
> -{
> -       int xgmi_tx_link, ret = 0;
> +               WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>
> -       switch (adev->asic_type) {
> -       case CHIP_VEGA20:
> -                       xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                               : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -                       if (xgmi_tx_link >= 0) {
> -                               ret = df_v3_6_stop_xgmi_link_cntr(adev,
> -                                                                 xgmi_tx_link,
> -                                                                 is_disable);
> -                               if (ret)
> -                                       return ret;
> -                       }
> -
> -                       ret = 0;
> -                       break;
> +               if (is_disable)
> +                       df_v3_6_pmc_release_cntr(adev, config);
> +
> +               break;
>         default:
>                 break;
>         }
> @@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>                                   uint64_t config,
>                                   uint64_t *count)
>  {
> -
> -       int xgmi_tx_link;
> +       uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> +       *count = 0;
>
>         switch (adev->asic_type) {
>         case CHIP_VEGA20:
> -               xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                                       : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>
> -               if (xgmi_tx_link >= 0) {
> -                       df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
> -                       df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
> -               }
> +               df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> +                                     &hi_base_addr);
> +
> +               if ((lo_base_addr == 0) || (hi_base_addr == 0))
> +                       return;
> +
> +               lo_val = RREG32_PCIE(lo_base_addr);
> +               hi_val = RREG32_PCIE(hi_base_addr);
> +
> +               *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> +
> +               if (*count >= DF_V3_6_PERFMON_OVERFLOW)
> +                       *count = 0;
> +
> +               DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +                       config, lo_base_addr, hi_base_addr, lo_val, hi_val);
>
>                 break;
> +
>         default:
>                 break;
>         }
> -
>  }
>
>  const struct amdgpu_df_funcs df_v3_6_funcs = {
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> index fcffd807764d..76998541bc30 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {
>  };
>
>  /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
> -#define AMDGPU_DF_MAX_COUNTERS         4
> +#define DF_V3_6_MAX_COUNTERS           4
>
>  /* get flags from df perfmon config */
> -#define GET_EVENT(x)                   (x & 0xFFUL)
> -#define GET_INSTANCE(x)                        ((x >> 8) & 0xFFUL)
> -#define GET_UNITMASK(x)                        ((x >> 16) & 0xFFUL)
> -#define GET_INSTANCE_CONFIG(x)         (0ULL | (0x07ULL) \
> -                                       | ((0x046ULL + x) << 8) \
> -                                       | (0x02 << 16))
> -
> -/* df event conf macros */
> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
> -               && GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
> -               && GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
> +#define DF_V3_6_GET_EVENT(x)           (x & 0xFFUL)
> +#define DF_V3_6_GET_INSTANCE(x)                ((x >> 8) & 0xFFUL)
> +#define DF_V3_6_GET_UNITMASK(x)                ((x >> 16) & 0xFFUL)
> +#define DF_V3_6_PERFMON_OVERFLOW       0xFFFFFFFFFFFFULL
>
> +extern const struct attribute_group *df_v3_6_attr_groups[];
>  extern const struct amdgpu_df_funcs df_v3_6_funcs;
>
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 78bd4fc07bab..7fee24ea7863 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
>  static int soc15_common_hw_init(void *handle)
>  {
>         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +       int ret;
>
>         /* enable pcie gen2/3 link */
>         soc15_pcie_gen3_enable(adev);
> @@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
>          */
>         if (adev->nbio_funcs->remap_hdp_registers)
>                 adev->nbio_funcs->remap_hdp_registers(adev);
> +
> +       adev->df_funcs->init(adev);
> +
>         /* enable the doorbell aperture */
>         soc15_enable_doorbell_aperture(adev, true);
>         /* HW doorbell routing policy: doorbell writing not
> --
> 2.17.1
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* RE: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found]     ` <CADnq5_O81Mmf=4N+EL8P6ARrbhAo8wXRVKZB78N5Dj7e6+upLw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
@ 2019-06-20 15:17       ` Kim, Jonathan
  2019-06-20 15:23       ` Kuehling, Felix
  1 sibling, 0 replies; 15+ messages in thread
From: Kim, Jonathan @ 2019-06-20 15:17 UTC (permalink / raw)
  To: Alex Deucher; +Cc: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

Sorry for the miss Alex.  I also did an early-merge by mistake before Felix had a chance to review some simplifications.

I just sent out a post merge fix patch for review to address comments.

Thanks,

Jon

-----Original Message-----
From: Alex Deucher <alexdeucher@gmail.com> 
Sent: Thursday, June 20, 2019 9:50 AM
To: Kim, Jonathan <Jonathan.Kim@amd.com>
Cc: amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons

[CAUTION: External Email]

On Wed, Jun 19, 2019 at 8:53 PM Kim, Jonathan <Jonathan.Kim@amd.com> wrote:
>
> v4: fixed kzalloc error check and modified df func init to return 
> error code
>
> v3: fixed cleanup by adding fini to free up adev df config counters
>
> v2: simplified by removing xgmi references in function names and 
> moving to generic df function names.  fixed issue by removing 
> hardcoded cake tx data events. streamlined error handling by having  
> df_v3_6_pmc_get_ctrl return error code.
>
> add pmu attribute groups and structures for perf events.
> add sysfs to track available df perfmon counters fix overflow handling 
> in perfmon counter reads.
>
> Change-Id: I61f731c0066b17834656c746e7efe038c4f62acf
> Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h  |  17 +  
> drivers/gpu/drm/amd/amdgpu/df_v3_6.c | 449 ++++++++++++---------------  
> drivers/gpu/drm/amd/amdgpu/df_v3_6.h |  19 +-
>  drivers/gpu/drm/amd/amdgpu/soc15.c   |   4 +
>  4 files changed, 231 insertions(+), 258 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index d355e9a09ad1..91cfcc7be5c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -732,6 +732,7 @@ struct amd_powerplay {  };
>
>  #define AMDGPU_RESET_MAGIC_NUM 64
> +#define AMDGPU_MAX_DF_PERFMONS 16
>  struct amdgpu_device {
>         struct device                   *dev;
>         struct drm_device               *ddev;
> @@ -962,6 +963,7 @@ struct amdgpu_device {
>         long                            compute_timeout;
>
>         uint64_t                        unique_id;
> +       uint64_t        df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
>  };
>
>  static inline struct amdgpu_device *amdgpu_ttm_adev(struct 
> ttm_bo_device *bdev) @@ -1201,4 +1203,19 @@ static inline int 
> amdgpu_dm_display_resume(struct amdgpu_device *adev) { return  #endif
>
>  #include "amdgpu_object.h"
> +
> +/* used by df_v3_6.c and amdgpu_pmu.c */
> +#define AMDGPU_PMU_ATTR(_name, _object)                                \
> +static ssize_t                                                         \
> +_name##_show(struct device *dev,                                       \
> +                              struct device_attribute *attr,           \
> +                              char *page)                              \
> +{                                                                      \
> +       BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1);                 \
> +       return sprintf(page, _object "\n");                             \
> +}                                                                      \
> +                                                                       
> +\ static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
> +
>  #endif
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c 
> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> index 8c09bf994acd..12e3e67013d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
> @@ -30,8 +30,104 @@
>  static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
>                                        16, 32, 0, 0, 0, 2, 4, 8};
>
> +/* init df format attrs */
> +AMDGPU_PMU_ATTR(event,         "config:0-7");
> +AMDGPU_PMU_ATTR(instance,      "config:8-15");
> +AMDGPU_PMU_ATTR(umask,         "config:16-23");
> +
> +/* df format attributes  */
> +static struct attribute *df_v3_6_format_attrs[] = {
> +       &pmu_attr_event.attr,
> +       &pmu_attr_instance.attr,
> +       &pmu_attr_umask.attr,
> +       NULL
> +};
> +
> +/* df format attribute group */
> +static struct attribute_group df_v3_6_format_attr_group = {
> +       .name = "format",
> +       .attrs = df_v3_6_format_attrs, };
> +
> +/* df event attrs */
> +AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
> +                     "event=0x7,instance=0x46,umask=0x2");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
> +                     "event=0x7,instance=0x47,umask=0x2");
> +AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
> +                     "event=0x7,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
> +                     "event=0x7,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
> +                     "event=0xb,instance=0x46,umask=0x4");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
> +                     "event=0xb,instance=0x47,umask=0x4");
> +AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
> +                     "event=0xb,instance=0x46,umask=0x8");
> +AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
> +                     "event=0xb,instance=0x47,umask=0x8");
> +
> +/* df event attributes  */
> +static struct attribute *df_v3_6_event_attrs[] = {
> +       &pmu_attr_cake0_pcsout_txdata.attr,
> +       &pmu_attr_cake1_pcsout_txdata.attr,
> +       &pmu_attr_cake0_pcsout_txmeta.attr,
> +       &pmu_attr_cake1_pcsout_txmeta.attr,
> +       &pmu_attr_cake0_ftiinstat_reqalloc.attr,
> +       &pmu_attr_cake1_ftiinstat_reqalloc.attr,
> +       &pmu_attr_cake0_ftiinstat_rspalloc.attr,
> +       &pmu_attr_cake1_ftiinstat_rspalloc.attr,
> +       NULL
> +};
> +
> +/* df event attribute group */
> +static struct attribute_group df_v3_6_event_attr_group = {
> +       .name = "events",
> +       .attrs = df_v3_6_event_attrs
> +};
> +
> +/* df event attr groups  */
> +const struct attribute_group *df_v3_6_attr_groups[] = {
> +               &df_v3_6_format_attr_group,
> +               &df_v3_6_event_attr_group,
> +               NULL
> +};
> +
> +/* get the number of df counters available */ static ssize_t 
> +df_v3_6_get_df_cntr_avail(struct device *dev,
> +               struct device_attribute *attr,
> +               char *buf)
> +{
> +       struct amdgpu_device *adev;
> +       struct drm_device *ddev;
> +       int i, count;
> +
> +       ddev = dev_get_drvdata(dev);
> +       adev = ddev->dev_private;
> +       count = 0;
> +
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if (adev->df_perfmon_config_assign_mask[i] == 0)
> +                       count++;
> +       }
> +
> +       return snprintf(buf, PAGE_SIZE, "%i\n", count); }
> +
> +/* device attr for available perfmon counters */ static 
> +DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
> +
> +/* init perfmons */
>  static void df_v3_6_init(struct amdgpu_device *adev)  {
> +       int i, ret;
> +
> +       ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
> +       if (ret)
> +               DRM_ERROR("failed to create file for available df 
> + counters\n");
> +
> +       for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
> +               adev->df_perfmon_config_assign_mask[i] = 0;
>  }

You never addressed my comments about df init from before.  df_init gets called from hw_init, so you shouldn't create files.  Move df init to being called from soc15 sw_init/fini and probably rename the df callbacks to sw_init and sw_fini so we know not to add hw state to those functions.  As is now, we'll end up needlessly creating and destorying files on suspend/resume.

Alex


>
>  static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev, 
> @@ -105,28 +201,19 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
>                 *flags |= AMD_CG_SUPPORT_DF_MGCG;  }
>
> -/* hold counter assignment per gpu struct */ -struct 
> df_v3_6_event_mask {
> -               struct amdgpu_device gpu;
> -               uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
> -};
> -
>  /* get assigned df perfmon ctr as int */ -static void 
> df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> -                                     uint64_t config,
> -                                     int *counter)
> +static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
> +                                     uint64_t config)
>  {
> -       struct df_v3_6_event_mask *mask;
>         int i;
>
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -       for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -               if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
> -                       *counter = i;
> -                       return;
> -               }
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if ((config & 0x0FFFFFFUL) ==
> +                                       adev->df_perfmon_config_assign_mask[i])
> +                       return i;
>         }
> +
> +       return -EINVAL;
>  }
>
>  /* get address based on counter assignment */ @@ -136,10 +223,7 @@ 
> static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
>                                  uint32_t *lo_base_addr,
>                                  uint32_t *hi_base_addr)  {
> -
> -       int target_cntr = -1;
> -
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +       int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr < 0)
>                 return;
> @@ -177,40 +261,38 @@ static void df_v3_6_pmc_get_read_settings(struct 
> amdgpu_device *adev,  }
>
>  /* get control counter settings i.e. address and values to set */ 
> -static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
> +static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
>                                           uint64_t config,
>                                           uint32_t *lo_base_addr,
>                                           uint32_t *hi_base_addr,
>                                           uint32_t *lo_val,
>                                           uint32_t *hi_val)  {
> -
> -       uint32_t eventsel, instance, unitmask;
> -       uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
> -
>         df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, 
> hi_base_addr);
>
> -       if (lo_val == NULL || hi_val == NULL)
> -               return;
> -
>         if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
> -               DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
> +               DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, 
> + Hi: %x",
>                                 *lo_base_addr, *hi_base_addr);
> -               return;
> +               return -ENXIO;
> +       }
> +
> +       if (lo_val && hi_val) {
> +               uint32_t eventsel, instance, unitmask;
> +               uint32_t instance_10, instance_5432, instance_76;
> +
> +               eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
> +               unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
> +               instance = DF_V3_6_GET_INSTANCE(config);
> +
> +               instance_10 = instance & 0x3;
> +               instance_5432 = (instance >> 2) & 0xf;
> +               instance_76 = (instance >> 6) & 0x3;
> +
> +               *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
> +               *hi_val = (instance_76 << 29) | instance_5432;
>         }
>
> -       eventsel = GET_EVENT(config);
> -       instance = GET_INSTANCE(config);
> -       unitmask = GET_UNITMASK(config);
> -
> -       es_5_0 = eventsel & 0x3FUL;
> -       es_13_6 = instance;
> -       es_13_0 = (es_13_6 << 6) + es_5_0;
> -       es_13_12 = (es_13_0 & 0x03000UL) >> 12;
> -       es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
> -       es_7_0 = es_13_0 & 0x0FFUL;
> -       *lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
> -       *hi_val = (es_11_8 | ((es_13_12)<<(29)));
> +       return 0;
>  }
>
>  /* assign df performance counters for read */ @@ -218,26 +300,21 @@ 
> static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
>                                    uint64_t config,
>                                    int *is_assigned)  {
> -
> -       struct df_v3_6_event_mask *mask;
>         int i, target_cntr;
>
> -       target_cntr = -1;
> -
>         *is_assigned = 0;
>
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> +       target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr >= 0) {
>                 *is_assigned = 1;
>                 return 0;
>         }
>
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> -
> -       for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
> -               if (mask->config_assign_mask[i] == 0ULL) {
> -                       mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
> +       for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
> +               if (adev->df_perfmon_config_assign_mask[i] == 0U) {
> +                       adev->df_perfmon_config_assign_mask[i] =
> +                                                       config & 
> + 0x0FFFFFFUL;
>                         return 0;
>                 }
>         }
> @@ -249,66 +326,17 @@ static int df_v3_6_pmc_assign_cntr(struct 
> amdgpu_device *adev,  static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
>                                      uint64_t config)  {
> -
> -       struct df_v3_6_event_mask *mask;
> -       int target_cntr;
> -
> -       target_cntr = -1;
> -
> -       df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
> -
> -       mask = container_of(adev, struct df_v3_6_event_mask, gpu);
> +       int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
>
>         if (target_cntr >= 0)
> -               mask->config_assign_mask[target_cntr] = 0ULL;
> -
> +               adev->df_perfmon_config_assign_mask[target_cntr] = 
> + 0ULL;
>  }
>
> -/*
> - * get xgmi link counters via programmable data fabric (df) counters 
> (max 4)
> - * using cake tx event.
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @count -> counters to pass
> - *
> - */
> -
> -static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                      int instance,
> -                                      uint64_t *count)
> -{
> -       uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -       uint64_t config;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
> -       df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> -                                     &hi_base_addr);
> -
> -       if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -               return;
> -
> -       lo_val = RREG32_PCIE(lo_base_addr);
> -       hi_val = RREG32_PCIE(hi_base_addr);
>
> -       *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> -}
> -
> -/*
> - * reset xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
> -static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                        int instance)
> +static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
> +                                        uint64_t config)
>  {
>         uint32_t lo_base_addr, hi_base_addr;
> -       uint64_t config;
> -
> -       config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
>
>         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
>                                       &hi_base_addr); @@ -320,185 
> +348,106 @@ static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
>         WREG32_PCIE(hi_base_addr, 0UL);  }
>
> -/*
> - * add xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - *
> - */
>
> -static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                     int instance)
> +static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
> +                                     uint64_t config)
>  {
>         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> -       uint64_t config;
>         int ret, is_assigned;
>
> -       if (instance < 0 || instance > 1)
> -               return -EINVAL;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
>         ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
>
>         if (ret || is_assigned)
>                 return ret;
>
> -       df_v3_6_pmc_get_ctrl_settings(adev,
> +       ret = df_v3_6_pmc_get_ctrl_settings(adev,
>                         config,
>                         &lo_base_addr,
>                         &hi_base_addr,
>                         &lo_val,
>                         &hi_val);
>
> +       if (ret)
> +               return ret;
> +
> +       DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +                       config, lo_base_addr, hi_base_addr, lo_val, 
> + hi_val);
> +
>         WREG32_PCIE(lo_base_addr, lo_val);
>         WREG32_PCIE(hi_base_addr, hi_val);
>
>         return ret;
>  }
>
> -
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either resume or assign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                       int instance,
> -                                       int is_enable)
> +static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> +                            int is_enable)
>  {
>         uint32_t lo_base_addr, hi_base_addr, lo_val;
> -       uint64_t config;
> -       int ret;
> -
> -       if (instance < 0 || instance > 1)
> -               return -EINVAL;
> -
> -       if (is_enable) {
> +       int ret = 0;
>
> -               ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
> -
> -               if (ret)
> -                       return ret;
> +       switch (adev->asic_type) {
> +       case CHIP_VEGA20:
>
> -       } else {
> +               df_v3_6_reset_perfmon_cntr(adev, config);
>
> -               config = GET_INSTANCE_CONFIG(instance);
> +               if (is_enable) {
> +                       ret = df_v3_6_add_perfmon_cntr(adev, config);
> +               } else {
> +                       ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +                                       config,
> +                                       &lo_base_addr,
> +                                       &hi_base_addr,
> +                                       NULL,
> +                                       NULL);
>
> -               df_v3_6_pmc_get_ctrl_settings(adev,
> -                               config,
> -                               &lo_base_addr,
> -                               &hi_base_addr,
> -                               NULL,
> -                               NULL);
> +                       if (ret)
> +                               return ret;
>
> -               if (lo_base_addr == 0)
> -                       return -EINVAL;
> +                       lo_val = RREG32_PCIE(lo_base_addr);
>
> -               lo_val = RREG32_PCIE(lo_base_addr);
> +                       DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +                               config, lo_base_addr, hi_base_addr, 
> + lo_val);
>
> -               WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +                       WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
> +               }
>
> -               ret = 0;
> +               break;
> +       default:
> +               break;
>         }
>
>         return ret;
> -
>  }
>
> -/*
> - * start xgmi link counters
> - *
> - * @adev -> amdgpu device
> - * @instance-> currently cake has 2 links to poll on vega20
> - * @is_enable -> either pause or unassign event via df perfmon
> - *
> - */
> -
> -static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
> -                                      int instance,
> -                                      int is_disable)
> +static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> +                           int is_disable)
>  {
> -
>         uint32_t lo_base_addr, hi_base_addr, lo_val;
> -       uint64_t config;
> -
> -       config = GET_INSTANCE_CONFIG(instance);
> -
> -       if (is_disable) {
> -               df_v3_6_reset_xgmi_link_cntr(adev, instance);
> -               df_v3_6_pmc_release_cntr(adev, config);
> -       } else {
> -
> -               df_v3_6_pmc_get_ctrl_settings(adev,
> -                               config,
> -                               &lo_base_addr,
> -                               &hi_base_addr,
> -                               NULL,
> -                               NULL);
> -
> -               if ((lo_base_addr == 0) || (hi_base_addr == 0))
> -                       return -EINVAL;
> -
> -               lo_val = RREG32_PCIE(lo_base_addr);
> -
> -               WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
> -       }
> -
> -       return 0;
> -}
> -
> -static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
> -                            int is_enable)
> -{
> -       int xgmi_tx_link, ret = 0;
> +       int ret = 0;
>
>         switch (adev->asic_type) {
>         case CHIP_VEGA20:
> -               xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                                       : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -               if (xgmi_tx_link >= 0)
> -                       ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
> -                                                     is_enable);
> +               ret = df_v3_6_pmc_get_ctrl_settings(adev,
> +                       config,
> +                       &lo_base_addr,
> +                       &hi_base_addr,
> +                       NULL,
> +                       NULL);
>
>                 if (ret)
>                         return ret;
>
> -               ret = 0;
> -               break;
> -       default:
> -               break;
> -       }
> +               lo_val = RREG32_PCIE(lo_base_addr);
>
> -       return ret;
> -}
> +               DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
> +                               config, lo_base_addr, hi_base_addr, 
> + lo_val);
>
> -static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
> -                           int is_disable)
> -{
> -       int xgmi_tx_link, ret = 0;
> +               WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
>
> -       switch (adev->asic_type) {
> -       case CHIP_VEGA20:
> -                       xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                               : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
> -
> -                       if (xgmi_tx_link >= 0) {
> -                               ret = df_v3_6_stop_xgmi_link_cntr(adev,
> -                                                                 xgmi_tx_link,
> -                                                                 is_disable);
> -                               if (ret)
> -                                       return ret;
> -                       }
> -
> -                       ret = 0;
> -                       break;
> +               if (is_disable)
> +                       df_v3_6_pmc_release_cntr(adev, config);
> +
> +               break;
>         default:
>                 break;
>         }
> @@ -510,24 +459,34 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
>                                   uint64_t config,
>                                   uint64_t *count)  {
> -
> -       int xgmi_tx_link;
> +       uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
> +       *count = 0;
>
>         switch (adev->asic_type) {
>         case CHIP_VEGA20:
> -               xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
> -                                       : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
>
> -               if (xgmi_tx_link >= 0) {
> -                       df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
> -                       df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
> -               }
> +               df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
> +                                     &hi_base_addr);
> +
> +               if ((lo_base_addr == 0) || (hi_base_addr == 0))
> +                       return;
> +
> +               lo_val = RREG32_PCIE(lo_base_addr);
> +               hi_val = RREG32_PCIE(hi_base_addr);
> +
> +               *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
> +
> +               if (*count >= DF_V3_6_PERFMON_OVERFLOW)
> +                       *count = 0;
> +
> +               DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
> +                       config, lo_base_addr, hi_base_addr, lo_val, 
> + hi_val);
>
>                 break;
> +
>         default:
>                 break;
>         }
> -
>  }
>
>  const struct amdgpu_df_funcs df_v3_6_funcs = { diff --git 
> a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h 
> b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> index fcffd807764d..76998541bc30 100644
> --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
> @@ -36,22 +36,15 @@ enum DF_V3_6_MGCG {  };
>
>  /* Defined in global_features.h as FTI_PERFMON_VISIBLE */
> -#define AMDGPU_DF_MAX_COUNTERS         4
> +#define DF_V3_6_MAX_COUNTERS           4
>
>  /* get flags from df perfmon config */
> -#define GET_EVENT(x)                   (x & 0xFFUL)
> -#define GET_INSTANCE(x)                        ((x >> 8) & 0xFFUL)
> -#define GET_UNITMASK(x)                        ((x >> 16) & 0xFFUL)
> -#define GET_INSTANCE_CONFIG(x)         (0ULL | (0x07ULL) \
> -                                       | ((0x046ULL + x) << 8) \
> -                                       | (0x02 << 16))
> -
> -/* df event conf macros */
> -#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
> -               && GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
> -#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
> -               && GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
> +#define DF_V3_6_GET_EVENT(x)           (x & 0xFFUL)
> +#define DF_V3_6_GET_INSTANCE(x)                ((x >> 8) & 0xFFUL)
> +#define DF_V3_6_GET_UNITMASK(x)                ((x >> 16) & 0xFFUL)
> +#define DF_V3_6_PERFMON_OVERFLOW       0xFFFFFFFFFFFFULL
>
> +extern const struct attribute_group *df_v3_6_attr_groups[];
>  extern const struct amdgpu_df_funcs df_v3_6_funcs;
>
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c 
> b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 78bd4fc07bab..7fee24ea7863 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -1066,6 +1066,7 @@ static void soc15_doorbell_range_init(struct 
> amdgpu_device *adev)  static int soc15_common_hw_init(void *handle)  {
>         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +       int ret;
>
>         /* enable pcie gen2/3 link */
>         soc15_pcie_gen3_enable(adev);
> @@ -1079,6 +1080,9 @@ static int soc15_common_hw_init(void *handle)
>          */
>         if (adev->nbio_funcs->remap_hdp_registers)
>                 adev->nbio_funcs->remap_hdp_registers(adev);
> +
> +       adev->df_funcs->init(adev);
> +
>         /* enable the doorbell aperture */
>         soc15_enable_doorbell_aperture(adev, true);
>         /* HW doorbell routing policy: doorbell writing not
> --
> 2.17.1
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons
       [not found]     ` <CADnq5_O81Mmf=4N+EL8P6ARrbhAo8wXRVKZB78N5Dj7e6+upLw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
  2019-06-20 15:17       ` Kim, Jonathan
@ 2019-06-20 15:23       ` Kuehling, Felix
  1 sibling, 0 replies; 15+ messages in thread
From: Kuehling, Felix @ 2019-06-20 15:23 UTC (permalink / raw)
  To: Alex Deucher, Kim, Jonathan
  Cc: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org

On 2019-06-20 9:49, Alex Deucher wrote:
> You never addressed my comments about df init from before.  df_init
> gets called from hw_init, so you shouldn't create files.  Move df init
> to being called from soc15 sw_init/fini and probably rename the df
> callbacks to sw_init and sw_fini so we know not to add hw state to
> those functions.  As is now, we'll end up needlessly creating and
> destorying files on suspend/resume.

Sorry I also missed that comment and the implications, or I would have 
pointed it out in my code review.

Jon's last patch looks mostly good to me and I think it addresses this 
with minimal fuss.

Regards,
   Felix

>
> Alex
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 15+ messages in thread

end of thread, other threads:[~2019-06-20 15:23 UTC | newest]

Thread overview: 15+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-06-19 22:50 [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons Kim, Jonathan
     [not found] ` <20190619224943.130022-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
2019-06-19 22:50   ` [PATCH 2/2] drm/amdgpu: add pmu counters Kim, Jonathan
     [not found]     ` <20190619224943.130022-2-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
2019-06-19 23:01       ` Kim, Jonathan
2019-06-19 23:23       ` Kuehling, Felix
2019-06-19 23:00   ` [PATCH 1/2] drm/amdgpu: update df_v3_6 for xgmi perfmons Kim, Jonathan
2019-06-19 23:27   ` Kuehling, Felix
     [not found]     ` <6cb35787-eff6-a354-3463-2fe3dd97c9f1-5C7GfCeVMHo@public.gmane.org>
2019-06-19 23:59       ` Kim, Jonathan
  -- strict thread matches above, loose matches on Subject: below --
2019-06-20  0:53 Kim, Jonathan
     [not found] ` <20190620005313.56371-1-jonathan.kim-5C7GfCeVMHo@public.gmane.org>
2019-06-20  0:55   ` Kim, Jonathan
2019-06-20  4:15   ` Kuehling, Felix
     [not found]     ` <84933584-80af-79ae-76ce-b2350e0aa69f-5C7GfCeVMHo@public.gmane.org>
2019-06-20  4:17       ` Kuehling, Felix
     [not found]         ` <5a8a03fe-af1a-370f-b4c5-5784cfcdc22c-5C7GfCeVMHo@public.gmane.org>
2019-06-20  5:21           ` Kim, Jonathan
2019-06-20 13:49   ` Alex Deucher
     [not found]     ` <CADnq5_O81Mmf=4N+EL8P6ARrbhAo8wXRVKZB78N5Dj7e6+upLw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2019-06-20 15:17       ` Kim, Jonathan
2019-06-20 15:23       ` Kuehling, Felix

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.