[RFC v1 0/4] Add support for drivers to decide bridge D3 policy

[RFC v1 0/4] Add support for drivers to decide bridge D3 policy

[Mario Limonciello]

The policy for whether PCI bridges are allowed to select D3 is dictated
by empirical results that are enumerated into pci_bridge_d3_possible().

In Windows this behaves differently in that Windows internal policy is
not used for devices when a power engine plugin driver provided by the
SOC vendor is installed.  This driver is used to decide the policy in
those cases.

This series implements a system that lets drivers register such a policy
control as well. It isn't activated for any SOCs by default.

This is heavily leveraged from the work in [1]

[1] https://lore.kernel.org/platform-driver-x86/20230906184354.45846-1-mario.limonciello@amd.com/
Mario Limonciello (4):
  ACPI: x86: s2idle: Export symbol for fetching constraints for module
    use
  PCI: Add support for drivers to decide bridge D3 policy
  PCI: Check for changes in pci_bridge_d3_possible() when updating D3
  platform/x86/amd: pmc: Add support for using constraints to decide D3
    policy

 drivers/acpi/x86/s2idle.c          |   1 +
 drivers/pci/pci.c                  | 148 +++++++++++++++++++++++++++--
 drivers/platform/x86/amd/pmc/pmc.c |  59 ++++++++++++
 include/linux/pci.h                |  54 +++++++++++
 4 files changed, 252 insertions(+), 10 deletions(-)

-- 
2.34.1

[RFC v1 1/4] ACPI: x86: s2idle: Export symbol for fetching constraints for module use

[Mario Limonciello]

The amd-pmc driver will be fetching constraints to make decisions at
suspend time. This driver can be compiled as a module, so export the
symbol for when it is a module.

Acked-by: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
---
 drivers/acpi/x86/s2idle.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c
index 08f7c6708206..de9c313c21fa 100644
--- a/drivers/acpi/x86/s2idle.c
+++ b/drivers/acpi/x86/s2idle.c
@@ -322,6 +322,7 @@ int acpi_get_lps0_constraint(struct acpi_device *adev)
 
 	return ACPI_STATE_UNKNOWN;
 }
+EXPORT_SYMBOL_GPL(acpi_get_lps0_constraint);
 
 static void lpi_check_constraints(void)
 {
-- 
2.34.1

[RFC v1 2/4] PCI: Add support for drivers to decide bridge D3 policy

[Mario Limonciello]

commit 9d26d3a8f1b0 ("PCI: Put PCIe ports into D3 during suspend")
changed pci_bridge_d3_possible() so that any vendor's PCIe ports
from modern machines (>=2015) are allowed to be put into D3.

This policy change has worked for most machines, but the behavior
is improved with `pcie_port_pm=off` on some others.

Add support for drivers to register a callback that they can optionally
use to decide the policy on supported machines.

A priority is used so that if multiple drivers register a callback and
support deciding for a device the highest priority driver will return
the value.

Suggested-by: Hans de Goede <hdegoede@redhat.com>
Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
---
 drivers/pci/pci.c   | 119 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  54 ++++++++++++++++++++
 2 files changed, 173 insertions(+)

diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 59c01d68c6d5..3b8e7b936908 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -21,6 +21,7 @@
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
+#include <linux/list_sort.h>
 #include <linux/log2.h>
 #include <linux/logic_pio.h>
 #include <linux/pm_wakeup.h>
@@ -54,6 +55,8 @@ unsigned int pci_pm_d3hot_delay;
 static void pci_pme_list_scan(struct work_struct *work);
 
 static LIST_HEAD(pci_pme_list);
+static LIST_HEAD(d3_possible_list);
+static DEFINE_MUTEX(d3_possible_list_mutex);
 static DEFINE_MUTEX(pci_pme_list_mutex);
 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
 
@@ -3031,6 +3034,16 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 		if (pci_bridge_d3_force)
 			return true;
 
+		/* check for any preferences for the bridge */
+		switch (bridge->driver_d3) {
+		case BRIDGE_PREF_DRIVER_D3:
+			return true;
+		case BRIDGE_PREF_DRIVER_D0:
+			return false;
+		default:
+			break;
+		}
+
 		/* Even the oldest 2010 Thunderbolt controller supports D3. */
 		if (bridge->is_thunderbolt)
 			return true;
@@ -3168,6 +3181,112 @@ void pci_d3cold_disable(struct pci_dev *dev)
 }
 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
 
+static struct pci_dev *pci_get_upstream_port(struct pci_dev *pci_dev)
+{
+	struct pci_dev *bridge;
+
+	bridge = pci_upstream_bridge(pci_dev);
+	if (!bridge)
+		return NULL;
+
+	if (!pci_is_pcie(bridge))
+		return NULL;
+
+	switch (pci_pcie_type(bridge)) {
+	case PCI_EXP_TYPE_ROOT_PORT:
+	case PCI_EXP_TYPE_UPSTREAM:
+	case PCI_EXP_TYPE_DOWNSTREAM:
+		return bridge;
+	default:
+		break;
+	};
+
+	return NULL;
+}
+
+static void pci_refresh_driver_d3(void)
+{
+	struct pci_d3_driver_ops *driver;
+	struct pci_dev *pci_dev = NULL;
+	struct pci_dev *bridge;
+
+	/* 1st pass: unset any preferences set a previous invocation */
+	while ((pci_dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pci_dev))) {
+		bridge = pci_get_upstream_port(pci_dev);
+		if (!bridge)
+			continue;
+
+		if (bridge->driver_d3 != BRIDGE_PREF_UNSET)
+			bridge->driver_d3 = BRIDGE_PREF_UNSET;
+	}
+
+	pci_dev = NULL;
+
+	/* 2nd pass: update the preference and refresh bridge_d3 */
+	while ((pci_dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pci_dev))) {
+		bridge = pci_get_upstream_port(pci_dev);
+		if (!bridge)
+			continue;
+
+		/* don't make multiple passes on the same device */
+		if (bridge->driver_d3 != BRIDGE_PREF_UNSET)
+			continue;
+
+		/* the list is pre-sorted by highest priority */
+		mutex_lock(&d3_possible_list_mutex);
+		list_for_each_entry(driver, &d3_possible_list, list_node) {
+			/* another higher priority driver already set preference */
+			if (bridge->driver_d3 != BRIDGE_PREF_UNSET)
+				break;
+			if (!driver->check)
+				continue;
+			bridge->driver_d3 = driver->check(bridge);
+		}
+		mutex_unlock(&d3_possible_list_mutex);
+
+		/* no driver set a preference */
+		if (bridge->driver_d3 == BRIDGE_PREF_UNSET)
+			bridge->driver_d3 = BRIDGE_PREF_NONE;
+
+		/* update bridge_d3 */
+		pci_bridge_d3_update(pci_dev);
+	}
+}
+
+static int pci_d3_driver_cmp(void *priv, const struct list_head *_a,
+			   const struct list_head *_b)
+{
+	struct pci_d3_driver_ops *a = container_of(_a, typeof(*a), list_node);
+	struct pci_d3_driver_ops *b = container_of(_b, typeof(*b), list_node);
+
+	if (a->priority < b->priority)
+		return -1;
+	else if (a->priority > b->priority)
+		return 1;
+	return 0;
+}
+
+int pci_register_driver_d3_policy_cb(struct pci_d3_driver_ops *arg)
+{
+	mutex_lock(&d3_possible_list_mutex);
+	list_add(&arg->list_node, &d3_possible_list);
+	list_sort(NULL, &d3_possible_list, pci_d3_driver_cmp);
+	mutex_unlock(&d3_possible_list_mutex);
+	pci_refresh_driver_d3();
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pci_register_driver_d3_policy_cb);
+
+void pci_unregister_driver_d3_policy_cb(struct pci_d3_driver_ops *arg)
+{
+	mutex_lock(&d3_possible_list_mutex);
+	list_del(&arg->list_node);
+	list_sort(NULL, &d3_possible_list, pci_d3_driver_cmp);
+	mutex_unlock(&d3_possible_list_mutex);
+	pci_refresh_driver_d3();
+}
+EXPORT_SYMBOL_GPL(pci_unregister_driver_d3_policy_cb);
+
 /**
  * pci_pm_init - Initialize PM functions of given PCI device
  * @dev: PCI device to handle.
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 8c7c2c3c6c65..c36903a4e351 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -389,6 +389,7 @@ struct pci_dev {
 						   bit manually */
 	unsigned int	d3hot_delay;	/* D3hot->D0 transition time in ms */
 	unsigned int	d3cold_delay;	/* D3cold->D0 transition time in ms */
+	unsigned int	driver_d3;	/* Driver D3 request preference */
 
 #ifdef CONFIG_PCIEASPM
 	struct pcie_link_state	*link_state;	/* ASPM link state */
@@ -1405,6 +1406,59 @@ bool pcie_relaxed_ordering_enabled(struct pci_dev *dev);
 void pci_resume_bus(struct pci_bus *bus);
 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state);
 
+/**
+ * enum bridge_d3_pref - D3 preference of a bridge
+ * @BRIDGE_PREF_UNSET: Not configured by driver
+ * @BRIDGE_PREF_NONE: Driver does not care
+ * @BRIDGE_PREF_DRIVER_D3: Driver prefers D3
+ * @BRIDGE_PREF_DRIVER_D0: Driver prefers D0
+ */
+enum bridge_d3_pref {
+	BRIDGE_PREF_UNSET,
+	BRIDGE_PREF_NONE,
+	BRIDGE_PREF_DRIVER_D3,
+	BRIDGE_PREF_DRIVER_D0,
+};
+
+/**
+ * pci_d3_driver_ops - Operations provided by a driver to evaluate D3 policy
+ * @list_node: the linked list node
+ * @priority: the priority of the callback
+ * @check: the callback to evaluate D3 policy
+ *
+ * For the callback drivers are expected to return:
+ *  BRIDGE_PREF_NONE if they can't evaluate the decision whether to support D3
+ *  BRIDGE_PREF_DRIVER_D0 if they decide the device should not support D3
+ *  BRIDGE_PREF_DRIVER_D3 if they decide the device should support D3
+ */
+struct pci_d3_driver_ops {
+	struct list_head list_node;
+	int priority;
+	enum bridge_d3_pref (*check)(struct pci_dev *pdev);
+};
+
+/**
+ * pci_register_driver_d3_policy_cb - Register a driver callback for configuring D3 policy
+ * @arg: driver provided structure with function pointer and priority
+ *
+ * This function can be used by drivers to register a callback that can be used
+ * to delegate the decision of whether a device should be used for D3 to that
+ * driver.
+ *
+ * Returns 0 on success, error code on failure.
+ */
+int pci_register_driver_d3_policy_cb(struct pci_d3_driver_ops *arg);
+
+/**
+ * pci_unregister_driver_d3_policy_cb - Unregister a driver callback for configuring D3 policy
+ * @arg: driver provided structure with function pointer and priority
+ *
+ * This function can be used by drivers to unregister a callback that can be used
+ * to delegate the decision of whether a device should be used for D3 to that
+ * driver.
+ */
+void pci_unregister_driver_d3_policy_cb(struct pci_d3_driver_ops *arg);
+
 /* For use by arch with custom probe code */
 void set_pcie_port_type(struct pci_dev *pdev);
 void set_pcie_hotplug_bridge(struct pci_dev *pdev);
-- 
2.34.1

Re: [RFC v1 2/4] PCI: Add support for drivers to decide bridge D3 policy

[Lukas Wunner]

On Mon, Oct 09, 2023 at 05:56:51PM -0500, Mario Limonciello wrote:
> commit 9d26d3a8f1b0 ("PCI: Put PCIe ports into D3 during suspend")
> changed pci_bridge_d3_possible() so that any vendor's PCIe ports
> from modern machines (>=2015) are allowed to be put into D3.
> 
> This policy change has worked for most machines, but the behavior
> is improved with `pcie_port_pm=off` on some others.
> 
> Add support for drivers to register a callback that they can optionally
> use to decide the policy on supported machines.

I would assume that drivers can decide the policy already today through
pci_d3cold_enable() / pci_d3cold_disable().

Why is this not sufficient and what's the benefit of the more complex
approach proposed here?

Thanks,

Lukas

Re: [RFC v1 2/4] PCI: Add support for drivers to decide bridge D3 policy

[Mario Limonciello]

On 10/14/2023 05:53, Lukas Wunner wrote:
> On Mon, Oct 09, 2023 at 05:56:51PM -0500, Mario Limonciello wrote:
>> commit 9d26d3a8f1b0 ("PCI: Put PCIe ports into D3 during suspend")
>> changed pci_bridge_d3_possible() so that any vendor's PCIe ports
>> from modern machines (>=2015) are allowed to be put into D3.
>>
>> This policy change has worked for most machines, but the behavior
>> is improved with `pcie_port_pm=off` on some others.
>>
>> Add support for drivers to register a callback that they can optionally
>> use to decide the policy on supported machines.
> 
> I would assume that drivers can decide the policy already today through
> pci_d3cold_enable() / pci_d3cold_disable().
> 
> Why is this not sufficient and what's the benefit of the more complex
> approach proposed here?
> 

This approach allows multiple drivers to participate.  Realistically 
however I don't know "many" drivers will participate in the first place.

 From our earlier conversations on the quirk thread I think your 
proposal will work as well for this, I'll try it.  Thanks.

[RFC v1 3/4] PCI: Check for changes in pci_bridge_d3_possible() when updating D3

[Mario Limonciello]

As drivers can report an optin or veto for a given PCI device it's
possible that pci_bridge_d3_possible() reports different values while
calling pci_bridge_d3_update().  Take these values into account while
updating the ability for a bridge to go into D3.

Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
---
 drivers/pci/pci.c | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 3b8e7b936908..77af444fcf1b 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -3094,6 +3094,14 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	return !*d3cold_ok;
 }
 
+static void pci_bridge_d3_propagate(struct pci_dev *bridge, bool d3_ok)
+{
+	if (bridge->bridge_d3 != d3_ok) {
+		bridge->bridge_d3 = d3_ok;
+		pci_bridge_d3_propagate(bridge, d3_ok);
+	}
+}
+
 /*
  * pci_bridge_d3_update - Update bridge D3 capabilities
  * @dev: PCI device which is changed
@@ -3106,12 +3114,16 @@ void pci_bridge_d3_update(struct pci_dev *dev)
 {
 	bool remove = !device_is_registered(&dev->dev);
 	struct pci_dev *bridge;
-	bool d3cold_ok = true;
+	bool d3_ok = true;
 
 	bridge = pci_upstream_bridge(dev);
-	if (!bridge || !pci_bridge_d3_possible(bridge))
+	if (!bridge)
 		return;
 
+	/* Propagate change to upstream bridges */
+	d3_ok = pci_bridge_d3_possible(bridge);
+	pci_bridge_d3_propagate(bridge, d3_ok);
+
 	/*
 	 * If D3 is currently allowed for the bridge, removing one of its
 	 * children won't change that.
@@ -3128,7 +3140,7 @@ void pci_bridge_d3_update(struct pci_dev *dev)
 	 * first may allow us to skip checking its siblings.
 	 */
 	if (!remove)
-		pci_dev_check_d3cold(dev, &d3cold_ok);
+		pci_dev_check_d3cold(dev, &d3_ok);
 
 	/*
 	 * If D3 is currently not allowed for the bridge, this may be caused
@@ -3136,15 +3148,12 @@ void pci_bridge_d3_update(struct pci_dev *dev)
 	 * so we need to go through all children to find out if one of them
 	 * continues to block D3.
 	 */
-	if (d3cold_ok && !bridge->bridge_d3)
+	if (d3_ok && !bridge->bridge_d3)
 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
-			     &d3cold_ok);
+			     &d3_ok);
 
-	if (bridge->bridge_d3 != d3cold_ok) {
-		bridge->bridge_d3 = d3cold_ok;
-		/* Propagate change to upstream bridges */
-		pci_bridge_d3_update(bridge);
-	}
+	/* Propagate change to upstream bridges */
+	pci_bridge_d3_propagate(bridge, d3_ok);
 }
 
 /**
-- 
2.34.1

[RFC v1 4/4] platform/x86/amd: pmc: Add support for using constraints to decide D3 policy

[Mario Limonciello]

The default kernel policy will allow modern machines to effectively put
all PCIe bridges into PCI D3. This policy doesn't match what Windows uses.

In Windows the driver stack includes a "Power Engine Plugin" (uPEP driver)
to decide the policy for integrated devices using PEP device constraints.

Device constraints are expressed as a number in the _DSM of the PNP0D80
device and exported by the kernel in acpi_get_lps0_constraint().

Add support for SoCs to use constraints on Linux as well for deciding
target state for integrated PCI bridges.

No SoCs are introduced by default with this change, they will be added
later on a case by case basis.

Link: https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/platform-design-for-modern-standby#low-power-core-silicon-cpu-soc-dram
Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
---
 drivers/platform/x86/amd/pmc/pmc.c | 59 ++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
index c1e788b67a74..34e76c6b3fb2 100644
--- a/drivers/platform/x86/amd/pmc/pmc.c
+++ b/drivers/platform/x86/amd/pmc/pmc.c
@@ -570,6 +570,14 @@ static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
 	debugfs_remove_recursive(dev->dbgfs_dir);
 }
 
+static bool amd_pmc_use_acpi_constraints(struct amd_pmc_dev *dev)
+{
+	switch (dev->cpu_id) {
+	default:
+		return false;
+	}
+}
+
 static bool amd_pmc_is_stb_supported(struct amd_pmc_dev *dev)
 {
 	switch (dev->cpu_id) {
@@ -741,6 +749,41 @@ static int amd_pmc_czn_wa_irq1(struct amd_pmc_dev *pdev)
 	return 0;
 }
 
+/*
+ * Constraints are specified in the ACPI LPS0 device and specify what the
+ * platform intended for the device.
+ *
+ * If a constraint is present and >= to ACPI_STATE_S3, then enable D3 for the
+ * device.
+ * If a constraint is not present or < ACPI_STATE_S3, then disable D3 for the
+ * device.
+ */
+static enum bridge_d3_pref amd_pmc_d3_check(struct pci_dev *pci_dev)
+{
+	struct acpi_device *adev = ACPI_COMPANION(&pci_dev->dev);
+	int constraint;
+
+	if (!adev)
+		return BRIDGE_PREF_UNSET;
+
+	constraint = acpi_get_lps0_constraint(adev);
+	dev_dbg(&pci_dev->dev, "constraint is %d\n", constraint);
+
+	switch (constraint) {
+	case ACPI_STATE_UNKNOWN:
+	case ACPI_STATE_S0:
+	case ACPI_STATE_S1:
+	case ACPI_STATE_S2:
+		return BRIDGE_PREF_DRIVER_D0;
+	case ACPI_STATE_S3:
+		return BRIDGE_PREF_DRIVER_D3;
+	default:
+		break;
+	}
+
+	return BRIDGE_PREF_UNSET;
+}
+
 static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
 {
 	struct rtc_device *rtc_device;
@@ -881,6 +924,11 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
 	.restore = amd_pmc_s2idle_restore,
 };
 
+static struct pci_d3_driver_ops amd_pmc_d3_ops = {
+	.check = amd_pmc_d3_check,
+	.priority = 50,
+};
+
 static int amd_pmc_suspend_handler(struct device *dev)
 {
 	struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
@@ -1074,10 +1122,19 @@ static int amd_pmc_probe(struct platform_device *pdev)
 			amd_pmc_quirks_init(dev);
 	}
 
+	if (amd_pmc_use_acpi_constraints(dev)) {
+		err = pci_register_driver_d3_policy_cb(&amd_pmc_d3_ops);
+		if (err)
+			goto err_register_lps0;
+	}
+
 	amd_pmc_dbgfs_register(dev);
 	pm_report_max_hw_sleep(U64_MAX);
 	return 0;
 
+err_register_lps0:
+	if (IS_ENABLED(CONFIG_SUSPEND))
+		acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
 err_pci_dev_put:
 	pci_dev_put(rdev);
 	return err;
@@ -1089,6 +1146,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
 
 	if (IS_ENABLED(CONFIG_SUSPEND))
 		acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
+	if (amd_pmc_use_acpi_constraints(dev))
+		pci_unregister_driver_d3_policy_cb(&amd_pmc_d3_ops);
 	amd_pmc_dbgfs_unregister(dev);
 	pci_dev_put(dev->rdev);
 	mutex_destroy(&dev->lock);
-- 
2.34.1

Re: [RFC v1 4/4] platform/x86/amd: pmc: Add support for using constraints to decide D3 policy

[Kai-Heng Feng]

Hi Mario,

On Tue, Oct 10, 2023 at 6:57 AM Mario Limonciello
<mario.limonciello@amd.com> wrote:
>
> The default kernel policy will allow modern machines to effectively put
> all PCIe bridges into PCI D3. This policy doesn't match what Windows uses.
>
> In Windows the driver stack includes a "Power Engine Plugin" (uPEP driver)
> to decide the policy for integrated devices using PEP device constraints.
>
> Device constraints are expressed as a number in the _DSM of the PNP0D80
> device and exported by the kernel in acpi_get_lps0_constraint().
>
> Add support for SoCs to use constraints on Linux as well for deciding
> target state for integrated PCI bridges.
>
> No SoCs are introduced by default with this change, they will be added
> later on a case by case basis.
>
> Link: https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/platform-design-for-modern-standby#low-power-core-silicon-cpu-soc-dram
> Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
> ---
>  drivers/platform/x86/amd/pmc/pmc.c | 59 ++++++++++++++++++++++++++++++
>  1 file changed, 59 insertions(+)
>
> diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
> index c1e788b67a74..34e76c6b3fb2 100644
> --- a/drivers/platform/x86/amd/pmc/pmc.c
> +++ b/drivers/platform/x86/amd/pmc/pmc.c
> @@ -570,6 +570,14 @@ static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
>         debugfs_remove_recursive(dev->dbgfs_dir);
>  }
>
> +static bool amd_pmc_use_acpi_constraints(struct amd_pmc_dev *dev)
> +{
> +       switch (dev->cpu_id) {
> +       default:
> +               return false;
> +       }
> +}
> +
>  static bool amd_pmc_is_stb_supported(struct amd_pmc_dev *dev)
>  {
>         switch (dev->cpu_id) {
> @@ -741,6 +749,41 @@ static int amd_pmc_czn_wa_irq1(struct amd_pmc_dev *pdev)
>         return 0;
>  }
>
> +/*
> + * Constraints are specified in the ACPI LPS0 device and specify what the
> + * platform intended for the device.
> + *
> + * If a constraint is present and >= to ACPI_STATE_S3, then enable D3 for the
> + * device.
> + * If a constraint is not present or < ACPI_STATE_S3, then disable D3 for the
> + * device.
> + */
> +static enum bridge_d3_pref amd_pmc_d3_check(struct pci_dev *pci_dev)
> +{
> +       struct acpi_device *adev = ACPI_COMPANION(&pci_dev->dev);
> +       int constraint;
> +
> +       if (!adev)
> +               return BRIDGE_PREF_UNSET;
> +
> +       constraint = acpi_get_lps0_constraint(adev);
> +       dev_dbg(&pci_dev->dev, "constraint is %d\n", constraint);
> +
> +       switch (constraint) {
> +       case ACPI_STATE_UNKNOWN:
> +       case ACPI_STATE_S0:
> +       case ACPI_STATE_S1:
> +       case ACPI_STATE_S2:

I believe it's a typo?
I think ACPI_STATE_Dx should be used for device state.

> +               return BRIDGE_PREF_DRIVER_D0;
> +       case ACPI_STATE_S3:
> +               return BRIDGE_PREF_DRIVER_D3;

I've seen both 3 (i.e. ACPI_STATE_D3_HOT) and 4 (i.e.
ACPI_STATE_D3_COLD) defined in LPI constraint table.
Should those two be treated differently?

> +       default:
> +               break;
> +       }
> +
> +       return BRIDGE_PREF_UNSET;
> +}
> +
>  static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
>  {
>         struct rtc_device *rtc_device;
> @@ -881,6 +924,11 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
>         .restore = amd_pmc_s2idle_restore,
>  };
>
> +static struct pci_d3_driver_ops amd_pmc_d3_ops = {
> +       .check = amd_pmc_d3_check,
> +       .priority = 50,
> +};
> +
>  static int amd_pmc_suspend_handler(struct device *dev)
>  {
>         struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
> @@ -1074,10 +1122,19 @@ static int amd_pmc_probe(struct platform_device *pdev)
>                         amd_pmc_quirks_init(dev);
>         }
>
> +       if (amd_pmc_use_acpi_constraints(dev)) {
> +               err = pci_register_driver_d3_policy_cb(&amd_pmc_d3_ops);
> +               if (err)
> +                       goto err_register_lps0;
> +       }

Does this only apply to PCI? USB can have ACPI companion too.

Kai-Heng

> +
>         amd_pmc_dbgfs_register(dev);
>         pm_report_max_hw_sleep(U64_MAX);
>         return 0;
>
> +err_register_lps0:
> +       if (IS_ENABLED(CONFIG_SUSPEND))
> +               acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
>  err_pci_dev_put:
>         pci_dev_put(rdev);
>         return err;
> @@ -1089,6 +1146,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
>
>         if (IS_ENABLED(CONFIG_SUSPEND))
>                 acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
> +       if (amd_pmc_use_acpi_constraints(dev))
> +               pci_unregister_driver_d3_policy_cb(&amd_pmc_d3_ops);
>         amd_pmc_dbgfs_unregister(dev);
>         pci_dev_put(dev->rdev);
>         mutex_destroy(&dev->lock);
> --
> 2.34.1
>

Re: [RFC v1 4/4] platform/x86/amd: pmc: Add support for using constraints to decide D3 policy

[Mario Limonciello]

On 10/15/2023 21:11, Kai-Heng Feng wrote:
> Hi Mario,
> 
> On Tue, Oct 10, 2023 at 6:57 AM Mario Limonciello
> <mario.limonciello@amd.com> wrote:
>>
>> The default kernel policy will allow modern machines to effectively put
>> all PCIe bridges into PCI D3. This policy doesn't match what Windows uses.
>>
>> In Windows the driver stack includes a "Power Engine Plugin" (uPEP driver)
>> to decide the policy for integrated devices using PEP device constraints.
>>
>> Device constraints are expressed as a number in the _DSM of the PNP0D80
>> device and exported by the kernel in acpi_get_lps0_constraint().
>>
>> Add support for SoCs to use constraints on Linux as well for deciding
>> target state for integrated PCI bridges.
>>
>> No SoCs are introduced by default with this change, they will be added
>> later on a case by case basis.
>>
>> Link: https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/platform-design-for-modern-standby#low-power-core-silicon-cpu-soc-dram
>> Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
>> ---
>>   drivers/platform/x86/amd/pmc/pmc.c | 59 ++++++++++++++++++++++++++++++
>>   1 file changed, 59 insertions(+)
>>
>> diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
>> index c1e788b67a74..34e76c6b3fb2 100644
>> --- a/drivers/platform/x86/amd/pmc/pmc.c
>> +++ b/drivers/platform/x86/amd/pmc/pmc.c
>> @@ -570,6 +570,14 @@ static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
>>          debugfs_remove_recursive(dev->dbgfs_dir);
>>   }
>>
>> +static bool amd_pmc_use_acpi_constraints(struct amd_pmc_dev *dev)
>> +{
>> +       switch (dev->cpu_id) {
>> +       default:
>> +               return false;
>> +       }
>> +}
>> +
>>   static bool amd_pmc_is_stb_supported(struct amd_pmc_dev *dev)
>>   {
>>          switch (dev->cpu_id) {
>> @@ -741,6 +749,41 @@ static int amd_pmc_czn_wa_irq1(struct amd_pmc_dev *pdev)
>>          return 0;
>>   }
>>
>> +/*
>> + * Constraints are specified in the ACPI LPS0 device and specify what the
>> + * platform intended for the device.
>> + *
>> + * If a constraint is present and >= to ACPI_STATE_S3, then enable D3 for the
>> + * device.
>> + * If a constraint is not present or < ACPI_STATE_S3, then disable D3 for the
>> + * device.
>> + */
>> +static enum bridge_d3_pref amd_pmc_d3_check(struct pci_dev *pci_dev)
>> +{
>> +       struct acpi_device *adev = ACPI_COMPANION(&pci_dev->dev);
>> +       int constraint;
>> +
>> +       if (!adev)
>> +               return BRIDGE_PREF_UNSET;
>> +
>> +       constraint = acpi_get_lps0_constraint(adev);
>> +       dev_dbg(&pci_dev->dev, "constraint is %d\n", constraint);
>> +
>> +       switch (constraint) {
>> +       case ACPI_STATE_UNKNOWN:
>> +       case ACPI_STATE_S0:
>> +       case ACPI_STATE_S1:
>> +       case ACPI_STATE_S2:
> 
> I believe it's a typo?
> I think ACPI_STATE_Dx should be used for device state.

Yes; typo thanks.

> 
>> +               return BRIDGE_PREF_DRIVER_D0;
>> +       case ACPI_STATE_S3:
>> +               return BRIDGE_PREF_DRIVER_D3;
> 
> I've seen both 3 (i.e. ACPI_STATE_D3_HOT) and 4 (i.e.
> ACPI_STATE_D3_COLD) defined in LPI constraint table.
> Should those two be treated differently?

Was this AMD system or Intel system?  AFAIK AMD doesn't use D3cold in 
constraints, they are collectively "D3".

> 
>> +       default:
>> +               break;
>> +       }
>> +
>> +       return BRIDGE_PREF_UNSET;
>> +}
>> +
>>   static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
>>   {
>>          struct rtc_device *rtc_device;
>> @@ -881,6 +924,11 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
>>          .restore = amd_pmc_s2idle_restore,
>>   };
>>
>> +static struct pci_d3_driver_ops amd_pmc_d3_ops = {
>> +       .check = amd_pmc_d3_check,
>> +       .priority = 50,
>> +};
>> +
>>   static int amd_pmc_suspend_handler(struct device *dev)
>>   {
>>          struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
>> @@ -1074,10 +1122,19 @@ static int amd_pmc_probe(struct platform_device *pdev)
>>                          amd_pmc_quirks_init(dev);
>>          }
>>
>> +       if (amd_pmc_use_acpi_constraints(dev)) {
>> +               err = pci_register_driver_d3_policy_cb(&amd_pmc_d3_ops);
>> +               if (err)
>> +                       goto err_register_lps0;
>> +       }
> 
> Does this only apply to PCI? USB can have ACPI companion too.

It only applies to things in the constraints, which is only "SOC 
internal" devices.  No internal USB devices.

> 
> Kai-Heng
> 
>> +
>>          amd_pmc_dbgfs_register(dev);
>>          pm_report_max_hw_sleep(U64_MAX);
>>          return 0;
>>
>> +err_register_lps0:
>> +       if (IS_ENABLED(CONFIG_SUSPEND))
>> +               acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
>>   err_pci_dev_put:
>>          pci_dev_put(rdev);
>>          return err;
>> @@ -1089,6 +1146,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
>>
>>          if (IS_ENABLED(CONFIG_SUSPEND))
>>                  acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
>> +       if (amd_pmc_use_acpi_constraints(dev))
>> +               pci_unregister_driver_d3_policy_cb(&amd_pmc_d3_ops);
>>          amd_pmc_dbgfs_unregister(dev);
>>          pci_dev_put(dev->rdev);
>>          mutex_destroy(&dev->lock);
>> --
>> 2.34.1
>>

Re: [RFC v1 4/4] platform/x86/amd: pmc: Add support for using constraints to decide D3 policy

[Kai-Heng Feng]

On Tue, Oct 17, 2023 at 5:34 AM Mario Limonciello
<mario.limonciello@amd.com> wrote:
>
> On 10/15/2023 21:11, Kai-Heng Feng wrote:
> > Hi Mario,
> >
> > On Tue, Oct 10, 2023 at 6:57 AM Mario Limonciello
> > <mario.limonciello@amd.com> wrote:
> >>
> >> The default kernel policy will allow modern machines to effectively put
> >> all PCIe bridges into PCI D3. This policy doesn't match what Windows uses.
> >>
> >> In Windows the driver stack includes a "Power Engine Plugin" (uPEP driver)
> >> to decide the policy for integrated devices using PEP device constraints.
> >>
> >> Device constraints are expressed as a number in the _DSM of the PNP0D80
> >> device and exported by the kernel in acpi_get_lps0_constraint().
> >>
> >> Add support for SoCs to use constraints on Linux as well for deciding
> >> target state for integrated PCI bridges.
> >>
> >> No SoCs are introduced by default with this change, they will be added
> >> later on a case by case basis.
> >>
> >> Link: https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/platform-design-for-modern-standby#low-power-core-silicon-cpu-soc-dram
> >> Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
> >> ---
> >>   drivers/platform/x86/amd/pmc/pmc.c | 59 ++++++++++++++++++++++++++++++
> >>   1 file changed, 59 insertions(+)
> >>
> >> diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
> >> index c1e788b67a74..34e76c6b3fb2 100644
> >> --- a/drivers/platform/x86/amd/pmc/pmc.c
> >> +++ b/drivers/platform/x86/amd/pmc/pmc.c
> >> @@ -570,6 +570,14 @@ static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
> >>          debugfs_remove_recursive(dev->dbgfs_dir);
> >>   }
> >>
> >> +static bool amd_pmc_use_acpi_constraints(struct amd_pmc_dev *dev)
> >> +{
> >> +       switch (dev->cpu_id) {
> >> +       default:
> >> +               return false;
> >> +       }
> >> +}
> >> +
> >>   static bool amd_pmc_is_stb_supported(struct amd_pmc_dev *dev)
> >>   {
> >>          switch (dev->cpu_id) {
> >> @@ -741,6 +749,41 @@ static int amd_pmc_czn_wa_irq1(struct amd_pmc_dev *pdev)
> >>          return 0;
> >>   }
> >>
> >> +/*
> >> + * Constraints are specified in the ACPI LPS0 device and specify what the
> >> + * platform intended for the device.
> >> + *
> >> + * If a constraint is present and >= to ACPI_STATE_S3, then enable D3 for the
> >> + * device.
> >> + * If a constraint is not present or < ACPI_STATE_S3, then disable D3 for the
> >> + * device.
> >> + */
> >> +static enum bridge_d3_pref amd_pmc_d3_check(struct pci_dev *pci_dev)
> >> +{
> >> +       struct acpi_device *adev = ACPI_COMPANION(&pci_dev->dev);
> >> +       int constraint;
> >> +
> >> +       if (!adev)
> >> +               return BRIDGE_PREF_UNSET;
> >> +
> >> +       constraint = acpi_get_lps0_constraint(adev);
> >> +       dev_dbg(&pci_dev->dev, "constraint is %d\n", constraint);
> >> +
> >> +       switch (constraint) {
> >> +       case ACPI_STATE_UNKNOWN:
> >> +       case ACPI_STATE_S0:
> >> +       case ACPI_STATE_S1:
> >> +       case ACPI_STATE_S2:
> >
> > I believe it's a typo?
> > I think ACPI_STATE_Dx should be used for device state.
>
> Yes; typo thanks.
>
> >
> >> +               return BRIDGE_PREF_DRIVER_D0;
> >> +       case ACPI_STATE_S3:
> >> +               return BRIDGE_PREF_DRIVER_D3;
> >
> > I've seen both 3 (i.e. ACPI_STATE_D3_HOT) and 4 (i.e.
> > ACPI_STATE_D3_COLD) defined in LPI constraint table.
> > Should those two be treated differently?
>
> Was this AMD system or Intel system?  AFAIK AMD doesn't use D3cold in
> constraints, they are collectively "D3".

Intel based system.

So the final D3 state is decided by the presence of power resources?

>
> >
> >> +       default:
> >> +               break;
> >> +       }
> >> +
> >> +       return BRIDGE_PREF_UNSET;
> >> +}
> >> +
> >>   static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
> >>   {
> >>          struct rtc_device *rtc_device;
> >> @@ -881,6 +924,11 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
> >>          .restore = amd_pmc_s2idle_restore,
> >>   };
> >>
> >> +static struct pci_d3_driver_ops amd_pmc_d3_ops = {
> >> +       .check = amd_pmc_d3_check,
> >> +       .priority = 50,
> >> +};
> >> +
> >>   static int amd_pmc_suspend_handler(struct device *dev)
> >>   {
> >>          struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
> >> @@ -1074,10 +1122,19 @@ static int amd_pmc_probe(struct platform_device *pdev)
> >>                          amd_pmc_quirks_init(dev);
> >>          }
> >>
> >> +       if (amd_pmc_use_acpi_constraints(dev)) {
> >> +               err = pci_register_driver_d3_policy_cb(&amd_pmc_d3_ops);
> >> +               if (err)
> >> +                       goto err_register_lps0;
> >> +       }
> >
> > Does this only apply to PCI? USB can have ACPI companion too.
>
> It only applies to things in the constraints, which is only "SOC
> internal" devices.  No internal USB devices.

So sounds like it only applies to PCI devices?

Kai-Heng

>
> >
> > Kai-Heng
> >
> >> +
> >>          amd_pmc_dbgfs_register(dev);
> >>          pm_report_max_hw_sleep(U64_MAX);
> >>          return 0;
> >>
> >> +err_register_lps0:
> >> +       if (IS_ENABLED(CONFIG_SUSPEND))
> >> +               acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
> >>   err_pci_dev_put:
> >>          pci_dev_put(rdev);
> >>          return err;
> >> @@ -1089,6 +1146,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
> >>
> >>          if (IS_ENABLED(CONFIG_SUSPEND))
> >>                  acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
> >> +       if (amd_pmc_use_acpi_constraints(dev))
> >> +               pci_unregister_driver_d3_policy_cb(&amd_pmc_d3_ops);
> >>          amd_pmc_dbgfs_unregister(dev);
> >>          pci_dev_put(dev->rdev);
> >>          mutex_destroy(&dev->lock);
> >> --
> >> 2.34.1
> >>
>

Re: [RFC v1 4/4] platform/x86/amd: pmc: Add support for using constraints to decide D3 policy

[Mario Limonciello]

>>> I've seen both 3 (i.e. ACPI_STATE_D3_HOT) and 4 (i.e.
>>> ACPI_STATE_D3_COLD) defined in LPI constraint table.
>>> Should those two be treated differently?
>>
>> Was this AMD system or Intel system?  AFAIK AMD doesn't use D3cold in
>> constraints, they are collectively "D3".
> 
> Intel based system.
> 
> So the final D3 state is decided by the presence of power resources?
> 

Right.  This is why I've mentioned in some of my series that 
pci_d3cold_enable() / pci_d3cold_disable() are misnamed and 
dev->no_d3cold can be misleading.

FYI - I have (slowly) been modifying this series to use 
pci_d3_cold_enable()/pci_d3_cold_disable() instead of the extra overhead 
of the policy decision but I have some problems.

I anticipate future version will also modify pci_bridge_d3_update().

>>
>>>
>>>> +       default:
>>>> +               break;
>>>> +       }
>>>> +
>>>> +       return BRIDGE_PREF_UNSET;
>>>> +}
>>>> +
>>>>    static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
>>>>    {
>>>>           struct rtc_device *rtc_device;
>>>> @@ -881,6 +924,11 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
>>>>           .restore = amd_pmc_s2idle_restore,
>>>>    };
>>>>
>>>> +static struct pci_d3_driver_ops amd_pmc_d3_ops = {
>>>> +       .check = amd_pmc_d3_check,
>>>> +       .priority = 50,
>>>> +};
>>>> +
>>>>    static int amd_pmc_suspend_handler(struct device *dev)
>>>>    {
>>>>           struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
>>>> @@ -1074,10 +1122,19 @@ static int amd_pmc_probe(struct platform_device *pdev)
>>>>                           amd_pmc_quirks_init(dev);
>>>>           }
>>>>
>>>> +       if (amd_pmc_use_acpi_constraints(dev)) {
>>>> +               err = pci_register_driver_d3_policy_cb(&amd_pmc_d3_ops);
>>>> +               if (err)
>>>> +                       goto err_register_lps0;
>>>> +       }
>>>
>>> Does this only apply to PCI? USB can have ACPI companion too.
>>
>> It only applies to things in the constraints, which is only "SOC
>> internal" devices.  No internal USB devices.
> 
> So sounds like it only applies to PCI devices?

Correct.

> 
> Kai-Heng
> 
>>
>>>
>>> Kai-Heng
>>>
>>>> +
>>>>           amd_pmc_dbgfs_register(dev);
>>>>           pm_report_max_hw_sleep(U64_MAX);
>>>>           return 0;
>>>>
>>>> +err_register_lps0:
>>>> +       if (IS_ENABLED(CONFIG_SUSPEND))
>>>> +               acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
>>>>    err_pci_dev_put:
>>>>           pci_dev_put(rdev);
>>>>           return err;
>>>> @@ -1089,6 +1146,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
>>>>
>>>>           if (IS_ENABLED(CONFIG_SUSPEND))
>>>>                   acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
>>>> +       if (amd_pmc_use_acpi_constraints(dev))
>>>> +               pci_unregister_driver_d3_policy_cb(&amd_pmc_d3_ops);
>>>>           amd_pmc_dbgfs_unregister(dev);
>>>>           pci_dev_put(dev->rdev);
>>>>           mutex_destroy(&dev->lock);
>>>> --
>>>> 2.34.1
>>>>
>>