[PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Arkadiusz Kubalewski]

Fix issues when performing unordered unbind/bind of a kernel modules
which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
Currently only serialized bind/unbind of such use case works, fix
the issues and allow for unserialized kernel module bind order.

The issues are observed on the ice driver, i.e.,

$ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
$ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind

results in:

ice 0000:af:00.0: Removed PTP clock
BUG: kernel NULL pointer dereference, address: 0000000000000010
PF: supervisor read access in kernel mode
PF: error_code(0x0000) - not-present page
PGD 0 P4D 0 
Oops: 0000 [#1] PREEMPT SMP PTI
CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-queue_19th-Oct-2023-01625-g039e5d15e451 #1
Hardware name: Intel Corporation S2600STB/S2600STB, BIOS SE5C620.86B.02.01.0008.031920191559 03/19/2019
RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66 08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b 10 41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
 <TASK>
 ? __die+0x20/0x70
 ? page_fault_oops+0x76/0x170
 ? exc_page_fault+0x65/0x150
 ? asm_exc_page_fault+0x22/0x30
 ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
 ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
 dpll_msg_add_pin_parents+0x142/0x1d0
 dpll_pin_event_send+0x7d/0x150
 dpll_pin_on_pin_unregister+0x3f/0x100
 ice_dpll_deinit_pins+0xa1/0x230 [ice]
 ice_dpll_deinit+0x29/0xe0 [ice]
 ice_remove+0xcd/0x200 [ice]
 pci_device_remove+0x33/0xa0
 device_release_driver_internal+0x193/0x200
 unbind_store+0x9d/0xb0
 kernfs_fop_write_iter+0x128/0x1c0
 vfs_write+0x2bb/0x3e0
 ksys_write+0x5f/0xe0
 do_syscall_64+0x59/0x90
 ? filp_close+0x1b/0x30
 ? do_dup2+0x7d/0xd0
 ? syscall_exit_work+0x103/0x130
 ? syscall_exit_to_user_mode+0x22/0x40
 ? do_syscall_64+0x69/0x90
 ? syscall_exit_work+0x103/0x130
 ? syscall_exit_to_user_mode+0x22/0x40
 ? do_syscall_64+0x69/0x90
 entry_SYSCALL_64_after_hwframe+0x6e/0xd8
RIP: 0033:0x7fdc7d93eb97
Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
 </TASK>
Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr i2c_i801 ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache jbd2 sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
CR2: 0000000000000010

Arkadiusz Kubalewski (3):
  dpll: fix pin dump crash after module unbind
  dpll: fix pin dump crash for rebound module
  dpll: fix register pin with unregistered parent pin

 drivers/dpll/dpll_core.c    |  8 ++------
 drivers/dpll/dpll_core.h    |  4 ++--
 drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++---------------
 3 files changed, 26 insertions(+), 23 deletions(-)

-- 
2.38.1

[PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Arkadiusz Kubalewski]

Disallow dump of unregistered parent pins, it is possible when parent
pin and dpll device registerer kernel module instance unbinds, and
other kernel module instances of the same dpll device have pins
registered with the parent pin. The user can invoke a pin-dump but as
the parent was unregistered, thus shall not be accessed by the
userspace, prevent that by checking if parent pin is still registered.

Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
---
 drivers/dpll/dpll_netlink.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index a6dc3997bf5c..93fc6c4b8a78 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct dpll_pin *pin,
 		void *parent_priv;
 
 		ppin = ref->pin;
+		/*
+		 * dump parent only if it is registered, thus prevent crash on
+		 * pin dump called when driver which registered the pin unbinds
+		 * and different instance registered pin on that parent pin
+		 */
+		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
+			continue;
 		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
 		ret = ops->state_on_pin_get(pin,
 					    dpll_pin_on_pin_priv(ppin, pin),
-- 
2.38.1

[PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Arkadiusz Kubalewski]

When a kernel module is unbound but the pin resources were not entirely
freed (other kernel module instance have had kept the reference to that
pin), and kernel module is again bound, the pin properties would not be
updated (the properties are only assigned when memory for the pin is
allocated), prop pointer still points to the kernel module memory of
the kernel module which was deallocated on the unbind.

If the pin dump is invoked in this state, the result is a kernel crash.
Prevent the crash by storing persistent pin properties in dpll subsystem,
copy the content from the kernel module when pin is allocated, instead of
using memory of the kernel module.

Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
---
 drivers/dpll/dpll_core.c    |  4 ++--
 drivers/dpll/dpll_core.h    |  4 ++--
 drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
index 3568149b9562..4077b562ba3b 100644
--- a/drivers/dpll/dpll_core.c
+++ b/drivers/dpll/dpll_core.c
@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct module *module,
 		ret = -EINVAL;
 		goto err;
 	}
-	pin->prop = prop;
+	memcpy(&pin->prop, prop, sizeof(pin->prop));
 	refcount_set(&pin->refcount, 1);
 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
 	unsigned long i, stop;
 	int ret;
 
-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
 		return -EINVAL;
 
 	if (WARN_ON(!ops) ||
diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
index 5585873c5c1b..717f715015c7 100644
--- a/drivers/dpll/dpll_core.h
+++ b/drivers/dpll/dpll_core.h
@@ -44,7 +44,7 @@ struct dpll_device {
  * @module:		module of creator
  * @dpll_refs:		hold referencees to dplls pin was registered with
  * @parent_refs:	hold references to parent pins pin was registered with
- * @prop:		pointer to pin properties given by registerer
+ * @prop:		pin properties copied from the registerer
  * @rclk_dev_name:	holds name of device when pin can recover clock from it
  * @refcount:		refcount
  **/
@@ -55,7 +55,7 @@ struct dpll_pin {
 	struct module *module;
 	struct xarray dpll_refs;
 	struct xarray parent_refs;
-	const struct dpll_pin_properties *prop;
+	struct dpll_pin_properties prop;
 	refcount_t refcount;
 };
 
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index 93fc6c4b8a78..963bbbbe6660 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct dpll_pin *pin,
 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
 			  DPLL_A_PIN_PAD))
 		return -EMSGSIZE;
-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
 		if (!nest)
 			return -EMSGSIZE;
-		freq = pin->prop->freq_supported[fs].min;
+		freq = pin->prop.freq_supported[fs].min;
 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
 				  &freq, DPLL_A_PIN_PAD)) {
 			nla_nest_cancel(msg, nest);
 			return -EMSGSIZE;
 		}
-		freq = pin->prop->freq_supported[fs].max;
+		freq = pin->prop.freq_supported[fs].max;
 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
 				  &freq, DPLL_A_PIN_PAD)) {
 			nla_nest_cancel(msg, nest);
@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct dpll_pin *pin, u32 freq)
 {
 	int fs;
 
-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
-		if (freq >= pin->prop->freq_supported[fs].min &&
-		    freq <= pin->prop->freq_supported[fs].max)
+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
+		if (freq >= pin->prop.freq_supported[fs].min &&
+		    freq <= pin->prop.freq_supported[fs].max)
 			return true;
 	return false;
 }
@@ -403,7 +403,7 @@ static int
 dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
 		     struct netlink_ext_ack *extack)
 {
-	const struct dpll_pin_properties *prop = pin->prop;
+	const struct dpll_pin_properties *prop = &pin->prop;
 	struct dpll_pin_ref *ref;
 	int ret;
 
@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32 parent_idx,
 	int ret;
 
 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
-	      pin->prop->capabilities)) {
+	      pin->prop.capabilities)) {
 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
 		return -EOPNOTSUPP;
 	}
@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct dpll_pin *pin,
 	int ret;
 
 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
-	      pin->prop->capabilities)) {
+	      pin->prop.capabilities)) {
 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
 		return -EOPNOTSUPP;
 	}
@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct dpll_pin *pin,
 	int ret;
 
 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
-	      pin->prop->capabilities)) {
+	      pin->prop.capabilities)) {
 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
 		return -EOPNOTSUPP;
 	}
@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct dpll_device *dpll,
 	int ret;
 
 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
-	      pin->prop->capabilities)) {
+	      pin->prop.capabilities)) {
 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
 		return -EOPNOTSUPP;
 	}
@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr,
 	int ret;
 
 	phase_adj = nla_get_s32(phase_adj_attr);
-	if (phase_adj > pin->prop->phase_range.max ||
-	    phase_adj < pin->prop->phase_range.min) {
+	if (phase_adj > pin->prop.phase_range.max ||
+	    phase_adj < pin->prop.phase_range.min) {
 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
 				    "phase adjust value not supported");
 		return -EINVAL;
@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr *mod_name_attr,
 	unsigned long i;
 
 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
-		prop = pin->prop;
+		prop = &pin->prop;
 		cid_match = clock_id ? pin->clock_id == clock_id : true;
 		mod_match = mod_name_attr && module_name(pin->module) ?
 			!nla_strcmp(mod_name_attr,
-- 
2.38.1

[PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Arkadiusz Kubalewski]

In case of multiple kernel module instances using the same dpll device:
if only one registers dpll device, then only that one can register
directly connected pins with a dpll device. If unregistered parent
determines if the muxed pin can be register with it or not, it forces
serialized driver load order - first the driver instance which registers
the direct pins needs to be loaded, then the other instances could
register muxed type pins.

Allow registration of a pin with a parent even if the parent was not
yet registered, thus allow ability for unserialized driver instance
load order.
Do not WARN_ON notification for unregistered pin, which can be invoked
for described case, instead just return error.

Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
---
 drivers/dpll/dpll_core.c    | 4 ----
 drivers/dpll/dpll_netlink.c | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
index 4077b562ba3b..ae884b92d68c 100644
--- a/drivers/dpll/dpll_core.c
+++ b/drivers/dpll/dpll_core.c
@@ -28,8 +28,6 @@ static u32 dpll_xa_id;
 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
 #define ASSERT_DPLL_NOT_REGISTERED(d)	\
 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
-#define ASSERT_PIN_REGISTERED(p)	\
-	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
 
 struct dpll_device_registration {
 	struct list_head list;
@@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
 	    WARN_ON(!ops->state_on_pin_get) ||
 	    WARN_ON(!ops->direction_get))
 		return -EINVAL;
-	if (ASSERT_PIN_REGISTERED(parent))
-		return -EINVAL;
 
 	mutex_lock(&dpll_lock);
 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv);
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index 963bbbbe6660..ff430f43304f 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct dpll_pin *pin)
 	int ret = -ENOMEM;
 	void *hdr;
 
-	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
+	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
 		return -ENODEV;
 
 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
-- 
2.38.1

Re: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Przemek Kitszel]

On 11/8/23 11:32, Arkadiusz Kubalewski wrote:
> Disallow dump of unregistered parent pins, it is possible when parent
> pin and dpll device registerer kernel module instance unbinds, and
> other kernel module instances of the same dpll device have pins
> registered with the parent pin. The user can invoke a pin-dump but as
> the parent was unregistered, thus shall not be accessed by the
> userspace, prevent that by checking if parent pin is still registered.
> 
> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
> ---
>   drivers/dpll/dpll_netlink.c | 7 +++++++
>   1 file changed, 7 insertions(+)
> 
> diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
> index a6dc3997bf5c..93fc6c4b8a78 100644
> --- a/drivers/dpll/dpll_netlink.c
> +++ b/drivers/dpll/dpll_netlink.c
> @@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct dpll_pin *pin,
>   		void *parent_priv;
>   
>   		ppin = ref->pin;
> +		/*
> +		 * dump parent only if it is registered, thus prevent crash on
> +		 * pin dump called when driver which registered the pin unbinds
> +		 * and different instance registered pin on that parent pin
> +		 */
> +		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
> +			continue;

What if unregister/unbind would happen right [here]?
[here]

>   		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
>   		ret = ops->state_on_pin_get(pin,
>   					    dpll_pin_on_pin_priv(ppin, pin),

RE: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Kubalewski, Arkadiusz]

>From: Kitszel, Przemyslaw <przemyslaw.kitszel@intel.com>
>Sent: Wednesday, November 8, 2023 12:36 PM
>
>On 11/8/23 11:32, Arkadiusz Kubalewski wrote:
>> Disallow dump of unregistered parent pins, it is possible when parent
>> pin and dpll device registerer kernel module instance unbinds, and
>> other kernel module instances of the same dpll device have pins
>> registered with the parent pin. The user can invoke a pin-dump but as
>> the parent was unregistered, thus shall not be accessed by the
>> userspace, prevent that by checking if parent pin is still registered.
>>
>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>> ---
>>   drivers/dpll/dpll_netlink.c | 7 +++++++
>>   1 file changed, 7 insertions(+)
>>
>> diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>> index a6dc3997bf5c..93fc6c4b8a78 100644
>> --- a/drivers/dpll/dpll_netlink.c
>> +++ b/drivers/dpll/dpll_netlink.c
>> @@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct
>dpll_pin *pin,
>>   		void *parent_priv;
>>
>>   		ppin = ref->pin;
>> +		/*
>> +		 * dump parent only if it is registered, thus prevent crash on
>> +		 * pin dump called when driver which registered the pin unbinds
>> +		 * and different instance registered pin on that parent pin
>> +		 */
>> +		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
>> +			continue;
>
>What if unregister/unbind would happen right [here]?
>[here]

There is a "global" mutex lock which guards the pin/dpll registration and all
netlink requests. For netlink requests in this case it is acquired in the
dpll_pin_pre_doit(..), while all the dpll subsystem interaction from kernel
modules are guarded in dpll_core.c api functions with the same lock.

So after all this use case is protected, just "higher" in the stack.

Thank you!
Arkadiusz

>
>>   		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
>>   		ret = ops->state_on_pin_get(pin,
>>   					    dpll_pin_on_pin_priv(ppin, pin),

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com wrote:
>When a kernel module is unbound but the pin resources were not entirely
>freed (other kernel module instance have had kept the reference to that
>pin), and kernel module is again bound, the pin properties would not be
>updated (the properties are only assigned when memory for the pin is
>allocated), prop pointer still points to the kernel module memory of
>the kernel module which was deallocated on the unbind.
>
>If the pin dump is invoked in this state, the result is a kernel crash.
>Prevent the crash by storing persistent pin properties in dpll subsystem,
>copy the content from the kernel module when pin is allocated, instead of
>using memory of the kernel module.
>
>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>---
> drivers/dpll/dpll_core.c    |  4 ++--
> drivers/dpll/dpll_core.h    |  4 ++--
> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
> 3 files changed, 18 insertions(+), 18 deletions(-)
>
>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>index 3568149b9562..4077b562ba3b 100644
>--- a/drivers/dpll/dpll_core.c
>+++ b/drivers/dpll/dpll_core.c
>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct module *module,
> 		ret = -EINVAL;
> 		goto err;
> 	}
>-	pin->prop = prop;
>+	memcpy(&pin->prop, prop, sizeof(pin->prop));

Odd, you don't care about the pointer within this structure?


> 	refcount_set(&pin->refcount, 1);
> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
> 	unsigned long i, stop;
> 	int ret;
> 
>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
> 		return -EINVAL;
> 
> 	if (WARN_ON(!ops) ||
>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>index 5585873c5c1b..717f715015c7 100644
>--- a/drivers/dpll/dpll_core.h
>+++ b/drivers/dpll/dpll_core.h
>@@ -44,7 +44,7 @@ struct dpll_device {
>  * @module:		module of creator
>  * @dpll_refs:		hold referencees to dplls pin was registered with
>  * @parent_refs:	hold references to parent pins pin was registered with
>- * @prop:		pointer to pin properties given by registerer
>+ * @prop:		pin properties copied from the registerer
>  * @rclk_dev_name:	holds name of device when pin can recover clock from it
>  * @refcount:		refcount
>  **/
>@@ -55,7 +55,7 @@ struct dpll_pin {
> 	struct module *module;
> 	struct xarray dpll_refs;
> 	struct xarray parent_refs;
>-	const struct dpll_pin_properties *prop;
>+	struct dpll_pin_properties prop;
> 	refcount_t refcount;
> };
> 
>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>index 93fc6c4b8a78..963bbbbe6660 100644
>--- a/drivers/dpll/dpll_netlink.c
>+++ b/drivers/dpll/dpll_netlink.c
>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct dpll_pin *pin,
> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
> 			  DPLL_A_PIN_PAD))
> 		return -EMSGSIZE;
>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
> 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
> 		if (!nest)
> 			return -EMSGSIZE;
>-		freq = pin->prop->freq_supported[fs].min;
>+		freq = pin->prop.freq_supported[fs].min;
> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
> 				  &freq, DPLL_A_PIN_PAD)) {
> 			nla_nest_cancel(msg, nest);
> 			return -EMSGSIZE;
> 		}
>-		freq = pin->prop->freq_supported[fs].max;
>+		freq = pin->prop.freq_supported[fs].max;
> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
> 				  &freq, DPLL_A_PIN_PAD)) {
> 			nla_nest_cancel(msg, nest);
>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct dpll_pin *pin, u32 freq)
> {
> 	int fs;
> 
>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>-		if (freq >= pin->prop->freq_supported[fs].min &&
>-		    freq <= pin->prop->freq_supported[fs].max)
>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>+		if (freq >= pin->prop.freq_supported[fs].min &&
>+		    freq <= pin->prop.freq_supported[fs].max)
> 			return true;
> 	return false;
> }
>@@ -403,7 +403,7 @@ static int
> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
> 		     struct netlink_ext_ack *extack)
> {
>-	const struct dpll_pin_properties *prop = pin->prop;
>+	const struct dpll_pin_properties *prop = &pin->prop;
> 	struct dpll_pin_ref *ref;
> 	int ret;
> 
>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32 parent_idx,
> 	int ret;
> 
> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>-	      pin->prop->capabilities)) {
>+	      pin->prop.capabilities)) {
> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
> 		return -EOPNOTSUPP;
> 	}
>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct dpll_pin *pin,
> 	int ret;
> 
> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>-	      pin->prop->capabilities)) {
>+	      pin->prop.capabilities)) {
> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
> 		return -EOPNOTSUPP;
> 	}
>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct dpll_pin *pin,
> 	int ret;
> 
> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>-	      pin->prop->capabilities)) {
>+	      pin->prop.capabilities)) {
> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
> 		return -EOPNOTSUPP;
> 	}
>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct dpll_device *dpll,
> 	int ret;
> 
> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>-	      pin->prop->capabilities)) {
>+	      pin->prop.capabilities)) {
> 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
> 		return -EOPNOTSUPP;
> 	}
>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr,
> 	int ret;
> 
> 	phase_adj = nla_get_s32(phase_adj_attr);
>-	if (phase_adj > pin->prop->phase_range.max ||
>-	    phase_adj < pin->prop->phase_range.min) {
>+	if (phase_adj > pin->prop.phase_range.max ||
>+	    phase_adj < pin->prop.phase_range.min) {
> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
> 				    "phase adjust value not supported");
> 		return -EINVAL;
>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr *mod_name_attr,
> 	unsigned long i;
> 
> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>-		prop = pin->prop;
>+		prop = &pin->prop;
> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
> 		mod_match = mod_name_attr && module_name(pin->module) ?
> 			!nla_strcmp(mod_name_attr,
>-- 
>2.38.1
>

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Jiri Pirko]

Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com wrote:
>In case of multiple kernel module instances using the same dpll device:
>if only one registers dpll device, then only that one can register

They why you don't register in multiple instances? See mlx5 for a
reference.


>directly connected pins with a dpll device. If unregistered parent
>determines if the muxed pin can be register with it or not, it forces
>serialized driver load order - first the driver instance which registers
>the direct pins needs to be loaded, then the other instances could
>register muxed type pins.
>
>Allow registration of a pin with a parent even if the parent was not
>yet registered, thus allow ability for unserialized driver instance

Weird.


>load order.
>Do not WARN_ON notification for unregistered pin, which can be invoked
>for described case, instead just return error.
>
>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>---
> drivers/dpll/dpll_core.c    | 4 ----
> drivers/dpll/dpll_netlink.c | 2 +-
> 2 files changed, 1 insertion(+), 5 deletions(-)
>
>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>index 4077b562ba3b..ae884b92d68c 100644
>--- a/drivers/dpll/dpll_core.c
>+++ b/drivers/dpll/dpll_core.c
>@@ -28,8 +28,6 @@ static u32 dpll_xa_id;
> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>-#define ASSERT_PIN_REGISTERED(p)	\
>-	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
> 
> struct dpll_device_registration {
> 	struct list_head list;
>@@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
> 	    WARN_ON(!ops->state_on_pin_get) ||
> 	    WARN_ON(!ops->direction_get))
> 		return -EINVAL;
>-	if (ASSERT_PIN_REGISTERED(parent))
>-		return -EINVAL;
> 
> 	mutex_lock(&dpll_lock);
> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv);
>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>index 963bbbbe6660..ff430f43304f 100644
>--- a/drivers/dpll/dpll_netlink.c
>+++ b/drivers/dpll/dpll_netlink.c
>@@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct dpll_pin *pin)
> 	int ret = -ENOMEM;
> 	void *hdr;
> 
>-	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>+	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
> 		return -ENODEV;
> 
> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>-- 
>2.38.1
>

Re: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Jiri Pirko]

Wed, Nov 08, 2023 at 11:32:24AM CET, arkadiusz.kubalewski@intel.com wrote:
>Disallow dump of unregistered parent pins, it is possible when parent
>pin and dpll device registerer kernel module instance unbinds, and
>other kernel module instances of the same dpll device have pins
>registered with the parent pin. The user can invoke a pin-dump but as
>the parent was unregistered, thus shall not be accessed by the
>userspace, prevent that by checking if parent pin is still registered.
>
>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>---
> drivers/dpll/dpll_netlink.c | 7 +++++++
> 1 file changed, 7 insertions(+)
>
>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>index a6dc3997bf5c..93fc6c4b8a78 100644
>--- a/drivers/dpll/dpll_netlink.c
>+++ b/drivers/dpll/dpll_netlink.c
>@@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct dpll_pin *pin,
> 		void *parent_priv;
> 
> 		ppin = ref->pin;
>+		/*
>+		 * dump parent only if it is registered, thus prevent crash on
>+		 * pin dump called when driver which registered the pin unbinds
>+		 * and different instance registered pin on that parent pin

Read this sentence like 10 times, still don't get what you mean.
Shouldn't comments be easy to understand?


>+		 */
>+		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
>+			continue;
> 		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
> 		ret = ops->state_on_pin_get(pin,
> 					    dpll_pin_on_pin_priv(ppin, pin),
>-- 
>2.38.1
>

RE: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Wednesday, November 8, 2023 4:09 PM
>
>Wed, Nov 08, 2023 at 11:32:24AM CET, arkadiusz.kubalewski@intel.com wrote:
>>Disallow dump of unregistered parent pins, it is possible when parent
>>pin and dpll device registerer kernel module instance unbinds, and
>>other kernel module instances of the same dpll device have pins
>>registered with the parent pin. The user can invoke a pin-dump but as
>>the parent was unregistered, thus shall not be accessed by the
>>userspace, prevent that by checking if parent pin is still registered.
>>
>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>---
>> drivers/dpll/dpll_netlink.c | 7 +++++++
>> 1 file changed, 7 insertions(+)
>>
>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>index a6dc3997bf5c..93fc6c4b8a78 100644
>>--- a/drivers/dpll/dpll_netlink.c
>>+++ b/drivers/dpll/dpll_netlink.c
>>@@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct
>dpll_pin *pin,
>> 		void *parent_priv;
>>
>> 		ppin = ref->pin;
>>+		/*
>>+		 * dump parent only if it is registered, thus prevent crash on
>>+		 * pin dump called when driver which registered the pin unbinds
>>+		 * and different instance registered pin on that parent pin
>
>Read this sentence like 10 times, still don't get what you mean.
>Shouldn't comments be easy to understand?
>

Hi,

Hmm, wondering isn't it better to remove this comment at all?
If you think it is needed I will rephrase it somehow..

Thank you!
Arkadiusz

>
>>+		 */
>>+		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
>>+			continue;
>> 		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
>> 		ret = ops->state_on_pin_get(pin,
>> 					    dpll_pin_on_pin_priv(ppin, pin),
>>--
>>2.38.1
>>

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Wednesday, November 8, 2023 4:08 PM
>
>Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com wrote:
>>In case of multiple kernel module instances using the same dpll device:
>>if only one registers dpll device, then only that one can register
>
>They why you don't register in multiple instances? See mlx5 for a
>reference.
>

Every registration requires ops, but for our case only PF0 is able to
control dpll pins and device, thus only this can provide ops.
Basically without PF0, dpll is not able to be controlled, as well
as directly connected pins.

>
>>directly connected pins with a dpll device. If unregistered parent
>>determines if the muxed pin can be register with it or not, it forces
>>serialized driver load order - first the driver instance which
>>registers the direct pins needs to be loaded, then the other instances
>>could register muxed type pins.
>>
>>Allow registration of a pin with a parent even if the parent was not
>>yet registered, thus allow ability for unserialized driver instance
>
>Weird.
>

Yeah, this is issue only for MUX/parent pin part, couldn't find better
way, but it doesn't seem to break things around..

Thank you!
Arkadiusz

>
>>load order.
>>Do not WARN_ON notification for unregistered pin, which can be invoked
>>for described case, instead just return error.
>>
>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>functions")
>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>---
>> drivers/dpll/dpll_core.c    | 4 ----
>> drivers/dpll/dpll_netlink.c | 2 +-
>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>
>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>4077b562ba3b..ae884b92d68c 100644
>>--- a/drivers/dpll/dpll_core.c
>>+++ b/drivers/dpll/dpll_core.c
>>@@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>-#define ASSERT_PIN_REGISTERED(p)	\
>>-	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>
>> struct dpll_device_registration {
>> 	struct list_head list;
>>@@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent,
>struct dpll_pin *pin,
>> 	    WARN_ON(!ops->state_on_pin_get) ||
>> 	    WARN_ON(!ops->direction_get))
>> 		return -EINVAL;
>>-	if (ASSERT_PIN_REGISTERED(parent))
>>-		return -EINVAL;
>>
>> 	mutex_lock(&dpll_lock);
>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>--git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>963bbbbe6660..ff430f43304f 100644
>>--- a/drivers/dpll/dpll_netlink.c
>>+++ b/drivers/dpll/dpll_netlink.c
>>@@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>dpll_pin *pin)
>> 	int ret = -ENOMEM;
>> 	void *hdr;
>>
>>-	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>+	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>> 		return -ENODEV;
>>
>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>--
>>2.38.1
>>

Re: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Vadim Fedorenko]

On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
> Fix issues when performing unordered unbind/bind of a kernel modules
> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
> Currently only serialized bind/unbind of such use case works, fix
> the issues and allow for unserialized kernel module bind order.
> 
> The issues are observed on the ice driver, i.e.,
> 
> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
> 
> results in:
> 
> ice 0000:af:00.0: Removed PTP clock
> BUG: kernel NULL pointer dereference, address: 0000000000000010
> PF: supervisor read access in kernel mode
> PF: error_code(0x0000) - not-present page
> PGD 0 P4D 0
> Oops: 0000 [#1] PREEMPT SMP PTI
> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-queue_19th-Oct-2023-01625-g039e5d15e451 #1
> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS SE5C620.86B.02.01.0008.031920191559 03/19/2019
> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66 08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b 10 41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000) knlGS:0000000000000000
> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> PKRU: 55555554
> Call Trace:
>   <TASK>
>   ? __die+0x20/0x70
>   ? page_fault_oops+0x76/0x170
>   ? exc_page_fault+0x65/0x150
>   ? asm_exc_page_fault+0x22/0x30
>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>   dpll_msg_add_pin_parents+0x142/0x1d0
>   dpll_pin_event_send+0x7d/0x150
>   dpll_pin_on_pin_unregister+0x3f/0x100
>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>   ice_dpll_deinit+0x29/0xe0 [ice]
>   ice_remove+0xcd/0x200 [ice]
>   pci_device_remove+0x33/0xa0
>   device_release_driver_internal+0x193/0x200
>   unbind_store+0x9d/0xb0
>   kernfs_fop_write_iter+0x128/0x1c0
>   vfs_write+0x2bb/0x3e0
>   ksys_write+0x5f/0xe0
>   do_syscall_64+0x59/0x90
>   ? filp_close+0x1b/0x30
>   ? do_dup2+0x7d/0xd0
>   ? syscall_exit_work+0x103/0x130
>   ? syscall_exit_to_user_mode+0x22/0x40
>   ? do_syscall_64+0x69/0x90
>   ? syscall_exit_work+0x103/0x130
>   ? syscall_exit_to_user_mode+0x22/0x40
>   ? do_syscall_64+0x69/0x90
>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
> RIP: 0033:0x7fdc7d93eb97
> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>   </TASK>
> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr i2c_i801 ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache jbd2 sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
> CR2: 0000000000000010
> 
> Arkadiusz Kubalewski (3):
>    dpll: fix pin dump crash after module unbind
>    dpll: fix pin dump crash for rebound module
>    dpll: fix register pin with unregistered parent pin
> 
>   drivers/dpll/dpll_core.c    |  8 ++------
>   drivers/dpll/dpll_core.h    |  4 ++--
>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++---------------
>   3 files changed, 26 insertions(+), 23 deletions(-)
> 


I still don't get how can we end up with unregistered pin. And shouldn't
drivers do unregister of dpll/pin during release procedure? I thought it
was kind of agreement we reached while developing the subsystem.

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Vadim Fedorenko]

On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>> From: Jiri Pirko <jiri@resnulli.us>
>> Sent: Wednesday, November 8, 2023 4:08 PM
>>
>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com wrote:
>>> In case of multiple kernel module instances using the same dpll device:
>>> if only one registers dpll device, then only that one can register
>>
>> They why you don't register in multiple instances? See mlx5 for a
>> reference.
>>
> 
> Every registration requires ops, but for our case only PF0 is able to
> control dpll pins and device, thus only this can provide ops.
> Basically without PF0, dpll is not able to be controlled, as well
> as directly connected pins.
> 
But why do you need other pins then, if FP0 doesn't exist?

>>
>>> directly connected pins with a dpll device. If unregistered parent
>>> determines if the muxed pin can be register with it or not, it forces
>>> serialized driver load order - first the driver instance which
>>> registers the direct pins needs to be loaded, then the other instances
>>> could register muxed type pins.
>>>
>>> Allow registration of a pin with a parent even if the parent was not
>>> yet registered, thus allow ability for unserialized driver instance
>>
>> Weird.
>>
> 
> Yeah, this is issue only for MUX/parent pin part, couldn't find better
> way, but it doesn't seem to break things around..
> 

I just wonder how do you see the registration procedure? How can parent
pin exist if it's not registered? I believe you cannot get it through
DPLL API, then the only possible way is to create it within the same
driver code, which can be simply re-arranged. Am I wrong here?

> Thank you!
> Arkadiusz
> 
>>
>>> load order.
>>> Do not WARN_ON notification for unregistered pin, which can be invoked
>>> for described case, instead just return error.
>>>
>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>> functions")
>>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>> ---
>>> drivers/dpll/dpll_core.c    | 4 ----
>>> drivers/dpll/dpll_netlink.c | 2 +-
>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>
>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>> 4077b562ba3b..ae884b92d68c 100644
>>> --- a/drivers/dpll/dpll_core.c
>>> +++ b/drivers/dpll/dpll_core.c
>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>>
>>> struct dpll_device_registration {
>>> 	struct list_head list;
>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent,
>> struct dpll_pin *pin,
>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>> 	    WARN_ON(!ops->direction_get))
>>> 		return -EINVAL;
>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>> -		return -EINVAL;
>>>
>>> 	mutex_lock(&dpll_lock);
>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>> 963bbbbe6660..ff430f43304f 100644
>>> --- a/drivers/dpll/dpll_netlink.c
>>> +++ b/drivers/dpll/dpll_netlink.c
>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>> dpll_pin *pin)
>>> 	int ret = -ENOMEM;
>>> 	void *hdr;
>>>
>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>> 		return -ENODEV;
>>>
>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>> --
>>> 2.38.1
>>>

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Wednesday, November 8, 2023 3:30 PM
>
>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com wrote:
>>When a kernel module is unbound but the pin resources were not entirely
>>freed (other kernel module instance have had kept the reference to that
>>pin), and kernel module is again bound, the pin properties would not be
>>updated (the properties are only assigned when memory for the pin is
>>allocated), prop pointer still points to the kernel module memory of
>>the kernel module which was deallocated on the unbind.
>>
>>If the pin dump is invoked in this state, the result is a kernel crash.
>>Prevent the crash by storing persistent pin properties in dpll subsystem,
>>copy the content from the kernel module when pin is allocated, instead of
>>using memory of the kernel module.
>>
>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>---
>> drivers/dpll/dpll_core.c    |  4 ++--
>> drivers/dpll/dpll_core.h    |  4 ++--
>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>
>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>index 3568149b9562..4077b562ba3b 100644
>>--- a/drivers/dpll/dpll_core.c
>>+++ b/drivers/dpll/dpll_core.c
>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>module *module,
>> 		ret = -EINVAL;
>> 		goto err;
>> 	}
>>-	pin->prop = prop;
>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>
>Odd, you don't care about the pointer within this structure?
>

Well, true. Need a fix.
Wondering if copying idea is better than just assigning prop pointer on
each call to dpll_pin_get(..) function (when pin already exists)?

Thank you!
Arkadiusz

>
>> 	refcount_set(&pin->refcount, 1);
>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent,
>>struct dpll_pin *pin,
>> 	unsigned long i, stop;
>> 	int ret;
>>
>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>> 		return -EINVAL;
>>
>> 	if (WARN_ON(!ops) ||
>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>index 5585873c5c1b..717f715015c7 100644
>>--- a/drivers/dpll/dpll_core.h
>>+++ b/drivers/dpll/dpll_core.h
>>@@ -44,7 +44,7 @@ struct dpll_device {
>>  * @module:		module of creator
>>  * @dpll_refs:		hold referencees to dplls pin was registered with
>>  * @parent_refs:	hold references to parent pins pin was registered
>>with
>>- * @prop:		pointer to pin properties given by registerer
>>+ * @prop:		pin properties copied from the registerer
>>  * @rclk_dev_name:	holds name of device when pin can recover clock
>>from it
>>  * @refcount:		refcount
>>  **/
>>@@ -55,7 +55,7 @@ struct dpll_pin {
>> 	struct module *module;
>> 	struct xarray dpll_refs;
>> 	struct xarray parent_refs;
>>-	const struct dpll_pin_properties *prop;
>>+	struct dpll_pin_properties prop;
>> 	refcount_t refcount;
>> };
>>
>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>index 93fc6c4b8a78..963bbbbe6660 100644
>>--- a/drivers/dpll/dpll_netlink.c
>>+++ b/drivers/dpll/dpll_netlink.c
>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct
>>dpll_pin *pin,
>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
>> 			  DPLL_A_PIN_PAD))
>> 		return -EMSGSIZE;
>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>> 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
>> 		if (!nest)
>> 			return -EMSGSIZE;
>>-		freq = pin->prop->freq_supported[fs].min;
>>+		freq = pin->prop.freq_supported[fs].min;
>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
>> 				  &freq, DPLL_A_PIN_PAD)) {
>> 			nla_nest_cancel(msg, nest);
>> 			return -EMSGSIZE;
>> 		}
>>-		freq = pin->prop->freq_supported[fs].max;
>>+		freq = pin->prop.freq_supported[fs].max;
>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
>> 				  &freq, DPLL_A_PIN_PAD)) {
>> 			nla_nest_cancel(msg, nest);
>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct dpll_pin
>>*pin, u32 freq)
>> {
>> 	int fs;
>>
>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>-		    freq <= pin->prop->freq_supported[fs].max)
>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>+		    freq <= pin->prop.freq_supported[fs].max)
>> 			return true;
>> 	return false;
>> }
>>@@ -403,7 +403,7 @@ static int
>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>> 		     struct netlink_ext_ack *extack)
>> {
>>-	const struct dpll_pin_properties *prop = pin->prop;
>>+	const struct dpll_pin_properties *prop = &pin->prop;
>> 	struct dpll_pin_ref *ref;
>> 	int ret;
>>
>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32
>>parent_idx,
>> 	int ret;
>>
>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>-	      pin->prop->capabilities)) {
>>+	      pin->prop.capabilities)) {
>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>> 		return -EOPNOTSUPP;
>> 	}
>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct
>>dpll_pin *pin,
>> 	int ret;
>>
>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>-	      pin->prop->capabilities)) {
>>+	      pin->prop.capabilities)) {
>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>> 		return -EOPNOTSUPP;
>> 	}
>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>dpll_pin *pin,
>> 	int ret;
>>
>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>-	      pin->prop->capabilities)) {
>>+	      pin->prop.capabilities)) {
>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>> 		return -EOPNOTSUPP;
>> 	}
>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct
>>dpll_device *dpll,
>> 	int ret;
>>
>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>-	      pin->prop->capabilities)) {
>>+	      pin->prop.capabilities)) {
>> 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
>> 		return -EOPNOTSUPP;
>> 	}
>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct
>>nlattr *phase_adj_attr,
>> 	int ret;
>>
>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>-	if (phase_adj > pin->prop->phase_range.max ||
>>-	    phase_adj < pin->prop->phase_range.min) {
>>+	if (phase_adj > pin->prop.phase_range.max ||
>>+	    phase_adj < pin->prop.phase_range.min) {
>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>> 				    "phase adjust value not supported");
>> 		return -EINVAL;
>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>*mod_name_attr,
>> 	unsigned long i;
>>
>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>-		prop = pin->prop;
>>+		prop = &pin->prop;
>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>> 			!nla_strcmp(mod_name_attr,
>>--
>>2.38.1
>>

Re: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Jiri Pirko]

Thu, Nov 09, 2023 at 10:49:49AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Wednesday, November 8, 2023 4:09 PM
>>
>>Wed, Nov 08, 2023 at 11:32:24AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>Disallow dump of unregistered parent pins, it is possible when parent
>>>pin and dpll device registerer kernel module instance unbinds, and
>>>other kernel module instances of the same dpll device have pins
>>>registered with the parent pin. The user can invoke a pin-dump but as
>>>the parent was unregistered, thus shall not be accessed by the
>>>userspace, prevent that by checking if parent pin is still registered.
>>>
>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>---
>>> drivers/dpll/dpll_netlink.c | 7 +++++++
>>> 1 file changed, 7 insertions(+)
>>>
>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>index a6dc3997bf5c..93fc6c4b8a78 100644
>>>--- a/drivers/dpll/dpll_netlink.c
>>>+++ b/drivers/dpll/dpll_netlink.c
>>>@@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg, struct
>>dpll_pin *pin,
>>> 		void *parent_priv;
>>>
>>> 		ppin = ref->pin;
>>>+		/*
>>>+		 * dump parent only if it is registered, thus prevent crash on
>>>+		 * pin dump called when driver which registered the pin unbinds
>>>+		 * and different instance registered pin on that parent pin
>>
>>Read this sentence like 10 times, still don't get what you mean.
>>Shouldn't comments be easy to understand?
>>
>
>Hi,
>
>Hmm, wondering isn't it better to remove this comment at all?
>If you think it is needed I will rephrase it somehow..

I don't know if it is needed as I don't understand it :)
Just remove it.


>
>Thank you!
>Arkadiusz
>
>>
>>>+		 */
>>>+		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
>>>+			continue;
>>> 		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
>>> 		ret = ops->state_on_pin_get(pin,
>>> 					    dpll_pin_on_pin_priv(ppin, pin),
>>>--
>>>2.38.1
>>>
>

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Wednesday, November 8, 2023 3:30 PM
>>
>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>When a kernel module is unbound but the pin resources were not entirely
>>>freed (other kernel module instance have had kept the reference to that
>>>pin), and kernel module is again bound, the pin properties would not be
>>>updated (the properties are only assigned when memory for the pin is
>>>allocated), prop pointer still points to the kernel module memory of
>>>the kernel module which was deallocated on the unbind.
>>>
>>>If the pin dump is invoked in this state, the result is a kernel crash.
>>>Prevent the crash by storing persistent pin properties in dpll subsystem,
>>>copy the content from the kernel module when pin is allocated, instead of
>>>using memory of the kernel module.
>>>
>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>---
>>> drivers/dpll/dpll_core.c    |  4 ++--
>>> drivers/dpll/dpll_core.h    |  4 ++--
>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>
>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>index 3568149b9562..4077b562ba3b 100644
>>>--- a/drivers/dpll/dpll_core.c
>>>+++ b/drivers/dpll/dpll_core.c
>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>module *module,
>>> 		ret = -EINVAL;
>>> 		goto err;
>>> 	}
>>>-	pin->prop = prop;
>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>
>>Odd, you don't care about the pointer within this structure?
>>
>
>Well, true. Need a fix.
>Wondering if copying idea is better than just assigning prop pointer on
>each call to dpll_pin_get(..) function (when pin already exists)?

Not sure what do you mean. Examples please.


>
>Thank you!
>Arkadiusz
>
>>
>>> 	refcount_set(&pin->refcount, 1);
>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent,
>>>struct dpll_pin *pin,
>>> 	unsigned long i, stop;
>>> 	int ret;
>>>
>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>> 		return -EINVAL;
>>>
>>> 	if (WARN_ON(!ops) ||
>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>index 5585873c5c1b..717f715015c7 100644
>>>--- a/drivers/dpll/dpll_core.h
>>>+++ b/drivers/dpll/dpll_core.h
>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>  * @module:		module of creator
>>>  * @dpll_refs:		hold referencees to dplls pin was registered with
>>>  * @parent_refs:	hold references to parent pins pin was registered
>>>with
>>>- * @prop:		pointer to pin properties given by registerer
>>>+ * @prop:		pin properties copied from the registerer
>>>  * @rclk_dev_name:	holds name of device when pin can recover clock
>>>from it
>>>  * @refcount:		refcount
>>>  **/
>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>> 	struct module *module;
>>> 	struct xarray dpll_refs;
>>> 	struct xarray parent_refs;
>>>-	const struct dpll_pin_properties *prop;
>>>+	struct dpll_pin_properties prop;
>>> 	refcount_t refcount;
>>> };
>>>
>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>--- a/drivers/dpll/dpll_netlink.c
>>>+++ b/drivers/dpll/dpll_netlink.c
>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct
>>>dpll_pin *pin,
>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
>>> 			  DPLL_A_PIN_PAD))
>>> 		return -EMSGSIZE;
>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>> 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>> 		if (!nest)
>>> 			return -EMSGSIZE;
>>>-		freq = pin->prop->freq_supported[fs].min;
>>>+		freq = pin->prop.freq_supported[fs].min;
>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>> 			nla_nest_cancel(msg, nest);
>>> 			return -EMSGSIZE;
>>> 		}
>>>-		freq = pin->prop->freq_supported[fs].max;
>>>+		freq = pin->prop.freq_supported[fs].max;
>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>> 			nla_nest_cancel(msg, nest);
>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct dpll_pin
>>>*pin, u32 freq)
>>> {
>>> 	int fs;
>>>
>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>> 			return true;
>>> 	return false;
>>> }
>>>@@ -403,7 +403,7 @@ static int
>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>> 		     struct netlink_ext_ack *extack)
>>> {
>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>> 	struct dpll_pin_ref *ref;
>>> 	int ret;
>>>
>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32
>>>parent_idx,
>>> 	int ret;
>>>
>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>-	      pin->prop->capabilities)) {
>>>+	      pin->prop.capabilities)) {
>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>> 		return -EOPNOTSUPP;
>>> 	}
>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct
>>>dpll_pin *pin,
>>> 	int ret;
>>>
>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>-	      pin->prop->capabilities)) {
>>>+	      pin->prop.capabilities)) {
>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>> 		return -EOPNOTSUPP;
>>> 	}
>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>>dpll_pin *pin,
>>> 	int ret;
>>>
>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>-	      pin->prop->capabilities)) {
>>>+	      pin->prop.capabilities)) {
>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>> 		return -EOPNOTSUPP;
>>> 	}
>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct
>>>dpll_device *dpll,
>>> 	int ret;
>>>
>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>-	      pin->prop->capabilities)) {
>>>+	      pin->prop.capabilities)) {
>>> 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
>>> 		return -EOPNOTSUPP;
>>> 	}
>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct
>>>nlattr *phase_adj_attr,
>>> 	int ret;
>>>
>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>+	    phase_adj < pin->prop.phase_range.min) {
>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>> 				    "phase adjust value not supported");
>>> 		return -EINVAL;
>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>*mod_name_attr,
>>> 	unsigned long i;
>>>
>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>-		prop = pin->prop;
>>>+		prop = &pin->prop;
>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>> 			!nla_strcmp(mod_name_attr,
>>>--
>>>2.38.1
>>>
>

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Jiri Pirko]

Thu, Nov 09, 2023 at 10:59:04AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Wednesday, November 8, 2023 4:08 PM
>>
>>Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>In case of multiple kernel module instances using the same dpll device:
>>>if only one registers dpll device, then only that one can register
>>
>>They why you don't register in multiple instances? See mlx5 for a
>>reference.
>>
>
>Every registration requires ops, but for our case only PF0 is able to

What makes PF0 so special? Smell like broken FW design... Care to fix
it?


>control dpll pins and device, thus only this can provide ops.
>Basically without PF0, dpll is not able to be controlled, as well
>as directly connected pins.
>
>>
>>>directly connected pins with a dpll device. If unregistered parent
>>>determines if the muxed pin can be register with it or not, it forces
>>>serialized driver load order - first the driver instance which
>>>registers the direct pins needs to be loaded, then the other instances
>>>could register muxed type pins.
>>>
>>>Allow registration of a pin with a parent even if the parent was not
>>>yet registered, thus allow ability for unserialized driver instance
>>
>>Weird.
>>
>
>Yeah, this is issue only for MUX/parent pin part, couldn't find better
>way, but it doesn't seem to break things around..
>
>Thank you!
>Arkadiusz
>
>>
>>>load order.
>>>Do not WARN_ON notification for unregistered pin, which can be invoked
>>>for described case, instead just return error.
>>>
>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>functions")
>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>---
>>> drivers/dpll/dpll_core.c    | 4 ----
>>> drivers/dpll/dpll_netlink.c | 2 +-
>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>
>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>>4077b562ba3b..ae884b92d68c 100644
>>>--- a/drivers/dpll/dpll_core.c
>>>+++ b/drivers/dpll/dpll_core.c
>>>@@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>-#define ASSERT_PIN_REGISTERED(p)	\
>>>-	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>>
>>> struct dpll_device_registration {
>>> 	struct list_head list;
>>>@@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent,
>>struct dpll_pin *pin,
>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>> 	    WARN_ON(!ops->direction_get))
>>> 		return -EINVAL;
>>>-	if (ASSERT_PIN_REGISTERED(parent))
>>>-		return -EINVAL;
>>>
>>> 	mutex_lock(&dpll_lock);
>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>>--git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>>963bbbbe6660..ff430f43304f 100644
>>>--- a/drivers/dpll/dpll_netlink.c
>>>+++ b/drivers/dpll/dpll_netlink.c
>>>@@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>dpll_pin *pin)
>>> 	int ret = -ENOMEM;
>>> 	void *hdr;
>>>
>>>-	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>>+	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>> 		return -ENODEV;
>>>
>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>--
>>>2.38.1
>>>

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>Sent: Thursday, November 9, 2023 11:56 AM
>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>
>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>> From: Jiri Pirko <jiri@resnulli.us>
>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>
>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>> wrote:
>>>> In case of multiple kernel module instances using the same dpll device:
>>>> if only one registers dpll device, then only that one can register
>>>
>>> They why you don't register in multiple instances? See mlx5 for a
>>> reference.
>>>
>>
>> Every registration requires ops, but for our case only PF0 is able to
>> control dpll pins and device, thus only this can provide ops.
>> Basically without PF0, dpll is not able to be controlled, as well
>> as directly connected pins.
>>
>But why do you need other pins then, if FP0 doesn't exist?
>

In general we don't need them at that point, but this is a corner case,
where users for some reason decided to unbind PF 0, and I treat this state
as temporary, where dpll/pins controllability is temporarily broken.

The dpll at that point is not registered, all the direct pins are also
not registered, thus not available to the users.

When I do dump at that point there are still 3 pins present, one for each
PF, although they are all zombies - no parents as their parent pins are not
registered (as the other patch [1/3] prevents dump of pin parent if the
parent is not registered). Maybe we can remove the REGISTERED mark for all
the muxed pins, if all their parents have been unregistered, so they won't
be visible to the user at all. Will try to POC that.

>>>
>>>> directly connected pins with a dpll device. If unregistered parent
>>>> determines if the muxed pin can be register with it or not, it forces
>>>> serialized driver load order - first the driver instance which
>>>> registers the direct pins needs to be loaded, then the other instances
>>>> could register muxed type pins.
>>>>
>>>> Allow registration of a pin with a parent even if the parent was not
>>>> yet registered, thus allow ability for unserialized driver instance
>>>
>>> Weird.
>>>
>>
>> Yeah, this is issue only for MUX/parent pin part, couldn't find better
>> way, but it doesn't seem to break things around..
>>
>
>I just wonder how do you see the registration procedure? How can parent
>pin exist if it's not registered? I believe you cannot get it through
>DPLL API, then the only possible way is to create it within the same
>driver code, which can be simply re-arranged. Am I wrong here?
>

By "parent exist" I mean the parent pin exist in the dpll subsystem
(allocated on pins xa), but it doesn't mean it is available to the users,
as it might not be registered with a dpll device.

We have this 2 step init approach:
1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
2.1. dpll_pin_register(..) -> register with a dpll device
2.2. dpll_pin_on_pin_register -> register with a parent pin

Basically:
- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for its
  recovery clock pin,
- other PF's only do step 1 for the direct input pins (as they must get
  reference to those in order to register recovery clock pin with them),
  and steps: 1 & 2.2 for their recovery clock pin.


Thank you!
Arkadiusz

>> Thank you!
>> Arkadiusz
>>
>>>
>>>> load order.
>>>> Do not WARN_ON notification for unregistered pin, which can be invoked
>>>> for described case, instead just return error.
>>>>
>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>> functions")
>>>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>> ---
>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>
>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>>> 4077b562ba3b..ae884b92d68c 100644
>>>> --- a/drivers/dpll/dpll_core.c
>>>> +++ b/drivers/dpll/dpll_core.c
>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>>>
>>>> struct dpll_device_registration {
>>>> 	struct list_head list;
>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>*parent,
>>> struct dpll_pin *pin,
>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>> 	    WARN_ON(!ops->direction_get))
>>>> 		return -EINVAL;
>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>> -		return -EINVAL;
>>>>
>>>> 	mutex_lock(&dpll_lock);
>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>>> 963bbbbe6660..ff430f43304f 100644
>>>> --- a/drivers/dpll/dpll_netlink.c
>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>> dpll_pin *pin)
>>>> 	int ret = -ENOMEM;
>>>> 	void *hdr;
>>>>
>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>> 		return -ENODEV;
>>>>
>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>> --
>>>> 2.38.1
>>>>
>

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 2:20 PM
>
>Thu, Nov 09, 2023 at 10:59:04AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Wednesday, November 8, 2023 4:08 PM
>>>
>>>Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>In case of multiple kernel module instances using the same dpll device:
>>>>if only one registers dpll device, then only that one can register
>>>
>>>They why you don't register in multiple instances? See mlx5 for a
>>>reference.
>>>
>>
>>Every registration requires ops, but for our case only PF0 is able to
>
>What makes PF0 so special? Smell like broken FW design... Care to fix
>it?
>

Well, from my perspective FW design it is.
AFAIR this single point of control is somehow related to HW design and
security requirements back when it was designed.. Don't think this would
be doable anytime soon (if doable at all).

Thank you!
Arkadiusz

>
>>control dpll pins and device, thus only this can provide ops.
>>Basically without PF0, dpll is not able to be controlled, as well
>>as directly connected pins.
>>
>>>
>>>>directly connected pins with a dpll device. If unregistered parent
>>>>determines if the muxed pin can be register with it or not, it forces
>>>>serialized driver load order - first the driver instance which
>>>>registers the direct pins needs to be loaded, then the other instances
>>>>could register muxed type pins.
>>>>
>>>>Allow registration of a pin with a parent even if the parent was not
>>>>yet registered, thus allow ability for unserialized driver instance
>>>
>>>Weird.
>>>
>>
>>Yeah, this is issue only for MUX/parent pin part, couldn't find better
>>way, but it doesn't seem to break things around..
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>load order.
>>>>Do not WARN_ON notification for unregistered pin, which can be invoked
>>>>for described case, instead just return error.
>>>>
>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>functions")
>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>---
>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>
>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>>>4077b562ba3b..ae884b92d68c 100644
>>>>--- a/drivers/dpll/dpll_core.c
>>>>+++ b/drivers/dpll/dpll_core.c
>>>>@@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>>-#define ASSERT_PIN_REGISTERED(p)	\
>>>>-	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>>>
>>>> struct dpll_device_registration {
>>>> 	struct list_head list;
>>>>@@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>*parent,
>>>struct dpll_pin *pin,
>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>> 	    WARN_ON(!ops->direction_get))
>>>> 		return -EINVAL;
>>>>-	if (ASSERT_PIN_REGISTERED(parent))
>>>>-		return -EINVAL;
>>>>
>>>> 	mutex_lock(&dpll_lock);
>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>>>--git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>>>963bbbbe6660..ff430f43304f 100644
>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>@@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>>dpll_pin *pin)
>>>> 	int ret = -ENOMEM;
>>>> 	void *hdr;
>>>>
>>>>-	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>>>+	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>> 		return -ENODEV;
>>>>
>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>--
>>>>2.38.1
>>>>

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 2:19 PM
>
>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>
>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>When a kernel module is unbound but the pin resources were not entirely
>>>>freed (other kernel module instance have had kept the reference to that
>>>>pin), and kernel module is again bound, the pin properties would not be
>>>>updated (the properties are only assigned when memory for the pin is
>>>>allocated), prop pointer still points to the kernel module memory of
>>>>the kernel module which was deallocated on the unbind.
>>>>
>>>>If the pin dump is invoked in this state, the result is a kernel crash.
>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>subsystem,
>>>>copy the content from the kernel module when pin is allocated, instead
>>>>of
>>>>using memory of the kernel module.
>>>>
>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>---
>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>
>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>index 3568149b9562..4077b562ba3b 100644
>>>>--- a/drivers/dpll/dpll_core.c
>>>>+++ b/drivers/dpll/dpll_core.c
>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>module *module,
>>>> 		ret = -EINVAL;
>>>> 		goto err;
>>>> 	}
>>>>-	pin->prop = prop;
>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>
>>>Odd, you don't care about the pointer within this structure?
>>>
>>
>>Well, true. Need a fix.
>>Wondering if copying idea is better than just assigning prop pointer on
>>each call to dpll_pin_get(..) function (when pin already exists)?
>
>Not sure what do you mean. Examples please.
>

Sure,

Basically this change:

diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
index ae884b92d68c..06b72d5877c3 100644
--- a/drivers/dpll/dpll_core.c
+++ b/drivers/dpll/dpll_core.c
@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct module *module,
                    pos->pin_idx == pin_idx &&
                    pos->module == module) {
                        ret = pos;
+                       pos->prop = prop;
                        refcount_inc(&ret->refcount);
                        break;
                }

would replace whole of this patch changes, although seems a bit hacky.

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>> 	refcount_set(&pin->refcount, 1);
>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>*parent,
>>>>struct dpll_pin *pin,
>>>> 	unsigned long i, stop;
>>>> 	int ret;
>>>>
>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>> 		return -EINVAL;
>>>>
>>>> 	if (WARN_ON(!ops) ||
>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>index 5585873c5c1b..717f715015c7 100644
>>>>--- a/drivers/dpll/dpll_core.h
>>>>+++ b/drivers/dpll/dpll_core.h
>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>  * @module:		module of creator
>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>with
>>>>  * @parent_refs:	hold references to parent pins pin was registered
>>>>with
>>>>- * @prop:		pointer to pin properties given by registerer
>>>>+ * @prop:		pin properties copied from the registerer
>>>>  * @rclk_dev_name:	holds name of device when pin can recover clock
>>>>from it
>>>>  * @refcount:		refcount
>>>>  **/
>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>> 	struct module *module;
>>>> 	struct xarray dpll_refs;
>>>> 	struct xarray parent_refs;
>>>>-	const struct dpll_pin_properties *prop;
>>>>+	struct dpll_pin_properties prop;
>>>> 	refcount_t refcount;
>>>> };
>>>>
>>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct
>>>>dpll_pin *pin,
>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
>>>> 			  DPLL_A_PIN_PAD))
>>>> 		return -EMSGSIZE;
>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>> 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>> 		if (!nest)
>>>> 			return -EMSGSIZE;
>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>> 			nla_nest_cancel(msg, nest);
>>>> 			return -EMSGSIZE;
>>>> 		}
>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>> 			nla_nest_cancel(msg, nest);
>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>dpll_pin
>>>>*pin, u32 freq)
>>>> {
>>>> 	int fs;
>>>>
>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>> 			return true;
>>>> 	return false;
>>>> }
>>>>@@ -403,7 +403,7 @@ static int
>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>> 		     struct netlink_ext_ack *extack)
>>>> {
>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>> 	struct dpll_pin_ref *ref;
>>>> 	int ret;
>>>>
>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32
>>>>parent_idx,
>>>> 	int ret;
>>>>
>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>-	      pin->prop->capabilities)) {
>>>>+	      pin->prop.capabilities)) {
>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>> 		return -EOPNOTSUPP;
>>>> 	}
>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct
>>>>dpll_pin *pin,
>>>> 	int ret;
>>>>
>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>-	      pin->prop->capabilities)) {
>>>>+	      pin->prop.capabilities)) {
>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>> 		return -EOPNOTSUPP;
>>>> 	}
>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>>>dpll_pin *pin,
>>>> 	int ret;
>>>>
>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>-	      pin->prop->capabilities)) {
>>>>+	      pin->prop.capabilities)) {
>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>> 		return -EOPNOTSUPP;
>>>> 	}
>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct
>>>>dpll_device *dpll,
>>>> 	int ret;
>>>>
>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>-	      pin->prop->capabilities)) {
>>>>+	      pin->prop.capabilities)) {
>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
>>>> 		return -EOPNOTSUPP;
>>>> 	}
>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct
>>>>nlattr *phase_adj_attr,
>>>> 	int ret;
>>>>
>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>> 				    "phase adjust value not supported");
>>>> 		return -EINVAL;
>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>*mod_name_attr,
>>>> 	unsigned long i;
>>>>
>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>-		prop = pin->prop;
>>>>+		prop = &pin->prop;
>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>> 			!nla_strcmp(mod_name_attr,
>>>>--
>>>>2.38.1
>>>>
>>

RE: [PATCH net 1/3] dpll: fix pin dump crash after module unbind

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 2:18 PM
>
>Thu, Nov 09, 2023 at 10:49:49AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Wednesday, November 8, 2023 4:09 PM
>>>
>>>Wed, Nov 08, 2023 at 11:32:24AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>Disallow dump of unregistered parent pins, it is possible when parent
>>>>pin and dpll device registerer kernel module instance unbinds, and
>>>>other kernel module instances of the same dpll device have pins
>>>>registered with the parent pin. The user can invoke a pin-dump but as
>>>>the parent was unregistered, thus shall not be accessed by the
>>>>userspace, prevent that by checking if parent pin is still registered.
>>>>
>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>---
>>>> drivers/dpll/dpll_netlink.c | 7 +++++++
>>>> 1 file changed, 7 insertions(+)
>>>>
>>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>index a6dc3997bf5c..93fc6c4b8a78 100644
>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>@@ -328,6 +328,13 @@ dpll_msg_add_pin_parents(struct sk_buff *msg,
>>>>        struct dpll_pin *pin,
>>>> 		void *parent_priv;
>>>>
>>>> 		ppin = ref->pin;
>>>>+		/*
>>>>+		 * dump parent only if it is registered, thus prevent crash on
>>>>+		 * pin dump called when driver which registered the pin unbinds
>>>>+		 * and different instance registered pin on that parent pin
>>>
>>>Read this sentence like 10 times, still don't get what you mean.
>>>Shouldn't comments be easy to understand?
>>>
>>
>>Hi,
>>
>>Hmm, wondering isn't it better to remove this comment at all?
>>If you think it is needed I will rephrase it somehow..
>
>I don't know if it is needed as I don't understand it :)
>Just remove it.
>

Sure, will do.

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>+		 */
>>>>+		if (!xa_get_mark(&dpll_pin_xa, ppin->id, DPLL_REGISTERED))
>>>>+			continue;
>>>> 		parent_priv = dpll_pin_on_dpll_priv(dpll_ref->dpll, ppin);
>>>> 		ret = ops->state_on_pin_get(pin,
>>>> 					    dpll_pin_on_pin_priv(ppin, pin),
>>>>--
>>>>2.38.1
>>>>
>>

RE: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Kubalewski, Arkadiusz]

>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>Sent: Thursday, November 9, 2023 11:51 AM
>
>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>> Fix issues when performing unordered unbind/bind of a kernel modules
>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>> Currently only serialized bind/unbind of such use case works, fix
>> the issues and allow for unserialized kernel module bind order.
>>
>> The issues are observed on the ice driver, i.e.,
>>
>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>
>> results in:
>>
>> ice 0000:af:00.0: Removed PTP clock
>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>> PF: supervisor read access in kernel mode
>> PF: error_code(0x0000) - not-present page
>> PGD 0 P4D 0
>> Oops: 0000 [#1] PREEMPT SMP PTI
>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-
>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66
>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b 10
>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>knlGS:0000000000000000
>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>> PKRU: 55555554
>> Call Trace:
>>   <TASK>
>>   ? __die+0x20/0x70
>>   ? page_fault_oops+0x76/0x170
>>   ? exc_page_fault+0x65/0x150
>>   ? asm_exc_page_fault+0x22/0x30
>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>   dpll_pin_event_send+0x7d/0x150
>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>   ice_remove+0xcd/0x200 [ice]
>>   pci_device_remove+0x33/0xa0
>>   device_release_driver_internal+0x193/0x200
>>   unbind_store+0x9d/0xb0
>>   kernfs_fop_write_iter+0x128/0x1c0
>>   vfs_write+0x2bb/0x3e0
>>   ksys_write+0x5f/0xe0
>>   do_syscall_64+0x59/0x90
>>   ? filp_close+0x1b/0x30
>>   ? do_dup2+0x7d/0xd0
>>   ? syscall_exit_work+0x103/0x130
>>   ? syscall_exit_to_user_mode+0x22/0x40
>>   ? do_syscall_64+0x69/0x90
>>   ? syscall_exit_work+0x103/0x130
>>   ? syscall_exit_to_user_mode+0x22/0x40
>>   ? do_syscall_64+0x69/0x90
>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>> RIP: 0033:0x7fdc7d93eb97
>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e
>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0
>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>   </TASK>
>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio
>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c
>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod
>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr
>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr i2c_i801
>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache jbd2
>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
>> CR2: 0000000000000010
>>
>> Arkadiusz Kubalewski (3):
>>    dpll: fix pin dump crash after module unbind
>>    dpll: fix pin dump crash for rebound module
>>    dpll: fix register pin with unregistered parent pin
>>
>>   drivers/dpll/dpll_core.c    |  8 ++------
>>   drivers/dpll/dpll_core.h    |  4 ++--
>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++---------------
>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>
>
>
>I still don't get how can we end up with unregistered pin. And shouldn't
>drivers do unregister of dpll/pin during release procedure? I thought it
>was kind of agreement we reached while developing the subsystem.
>

It's definitely not about ending up with unregistered pins.

Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in opposite
order: PF3, PF2, PF1, PF0. And this is working without any issues.

Above crash is caused because of unordered driver unload, where dpll subsystem
tries to notify muxed pin was deleted, but at that time the parent is already
gone, thus data points to memory which is no longer available, thus crash
happens when trying to dump pin parents.

This series fixes all issues I could find connected to the situation where
muxed-pins are trying to access their parents, when parent registerer was removed
in the meantime.

Thank you!
Arkadiusz

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Jiri Pirko]

Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>Sent: Thursday, November 9, 2023 11:56 AM
>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>>
>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>
>>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>> wrote:
>>>>> In case of multiple kernel module instances using the same dpll device:
>>>>> if only one registers dpll device, then only that one can register
>>>>
>>>> They why you don't register in multiple instances? See mlx5 for a
>>>> reference.
>>>>
>>>
>>> Every registration requires ops, but for our case only PF0 is able to
>>> control dpll pins and device, thus only this can provide ops.
>>> Basically without PF0, dpll is not able to be controlled, as well
>>> as directly connected pins.
>>>
>>But why do you need other pins then, if FP0 doesn't exist?
>>
>
>In general we don't need them at that point, but this is a corner case,
>where users for some reason decided to unbind PF 0, and I treat this state
>as temporary, where dpll/pins controllability is temporarily broken.

So resolve this broken situation internally in the driver, registering
things only in case PF0 is present. Some simple notification infra would
do. Don't drag this into the subsystem internals.


>
>The dpll at that point is not registered, all the direct pins are also
>not registered, thus not available to the users.
>
>When I do dump at that point there are still 3 pins present, one for each
>PF, although they are all zombies - no parents as their parent pins are not
>registered (as the other patch [1/3] prevents dump of pin parent if the
>parent is not registered). Maybe we can remove the REGISTERED mark for all
>the muxed pins, if all their parents have been unregistered, so they won't
>be visible to the user at all. Will try to POC that.
>
>>>>
>>>>> directly connected pins with a dpll device. If unregistered parent
>>>>> determines if the muxed pin can be register with it or not, it forces
>>>>> serialized driver load order - first the driver instance which
>>>>> registers the direct pins needs to be loaded, then the other instances
>>>>> could register muxed type pins.
>>>>>
>>>>> Allow registration of a pin with a parent even if the parent was not
>>>>> yet registered, thus allow ability for unserialized driver instance
>>>>
>>>> Weird.
>>>>
>>>
>>> Yeah, this is issue only for MUX/parent pin part, couldn't find better
>>> way, but it doesn't seem to break things around..
>>>
>>
>>I just wonder how do you see the registration procedure? How can parent
>>pin exist if it's not registered? I believe you cannot get it through
>>DPLL API, then the only possible way is to create it within the same
>>driver code, which can be simply re-arranged. Am I wrong here?
>>
>
>By "parent exist" I mean the parent pin exist in the dpll subsystem
>(allocated on pins xa), but it doesn't mean it is available to the users,
>as it might not be registered with a dpll device.
>
>We have this 2 step init approach:
>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>2.1. dpll_pin_register(..) -> register with a dpll device
>2.2. dpll_pin_on_pin_register -> register with a parent pin
>
>Basically:
>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for its
>  recovery clock pin,
>- other PF's only do step 1 for the direct input pins (as they must get
>  reference to those in order to register recovery clock pin with them),
>  and steps: 1 & 2.2 for their recovery clock pin.
>
>
>Thank you!
>Arkadiusz
>
>>> Thank you!
>>> Arkadiusz
>>>
>>>>
>>>>> load order.
>>>>> Do not WARN_ON notification for unregistered pin, which can be invoked
>>>>> for described case, instead just return error.
>>>>>
>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>> functions")
>>>>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>> ---
>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>
>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index
>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>> --- a/drivers/dpll/dpll_core.c
>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))
>>>>>
>>>>> struct dpll_device_registration {
>>>>> 	struct list_head list;
>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>*parent,
>>>> struct dpll_pin *pin,
>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>> 	    WARN_ON(!ops->direction_get))
>>>>> 		return -EINVAL;
>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>> -		return -EINVAL;
>>>>>
>>>>> 	mutex_lock(&dpll_lock);
>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv); diff
>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index
>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>> dpll_pin *pin)
>>>>> 	int ret = -ENOMEM;
>>>>> 	void *hdr;
>>>>>
>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED)))
>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>> 		return -ENODEV;
>>>>>
>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>> --
>>>>> 2.38.1
>>>>>
>>
>

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Thursday, November 9, 2023 2:19 PM
>>
>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>
>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>When a kernel module is unbound but the pin resources were not entirely
>>>>>freed (other kernel module instance have had kept the reference to that
>>>>>pin), and kernel module is again bound, the pin properties would not be
>>>>>updated (the properties are only assigned when memory for the pin is
>>>>>allocated), prop pointer still points to the kernel module memory of
>>>>>the kernel module which was deallocated on the unbind.
>>>>>
>>>>>If the pin dump is invoked in this state, the result is a kernel crash.
>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>subsystem,
>>>>>copy the content from the kernel module when pin is allocated, instead
>>>>>of
>>>>>using memory of the kernel module.
>>>>>
>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base functions")
>>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>---
>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>
>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>>module *module,
>>>>> 		ret = -EINVAL;
>>>>> 		goto err;
>>>>> 	}
>>>>>-	pin->prop = prop;
>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>
>>>>Odd, you don't care about the pointer within this structure?
>>>>
>>>
>>>Well, true. Need a fix.
>>>Wondering if copying idea is better than just assigning prop pointer on
>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>
>>Not sure what do you mean. Examples please.
>>
>
>Sure,
>
>Basically this change:
>
>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>index ae884b92d68c..06b72d5877c3 100644
>--- a/drivers/dpll/dpll_core.c
>+++ b/drivers/dpll/dpll_core.c
>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct module *module,
>                    pos->pin_idx == pin_idx &&
>                    pos->module == module) {
>                        ret = pos;
>+                       pos->prop = prop;
>                        refcount_inc(&ret->refcount);
>                        break;
>                }
>
>would replace whole of this patch changes, although seems a bit hacky.

Or event better, as I suggested in the other patch reply, resolve this
internally in the driver registering things only when they are valid.
Much better then to hack anything in dpll core.


>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>> 	refcount_set(&pin->refcount, 1);
>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>*parent,
>>>>>struct dpll_pin *pin,
>>>>> 	unsigned long i, stop;
>>>>> 	int ret;
>>>>>
>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>> 		return -EINVAL;
>>>>>
>>>>> 	if (WARN_ON(!ops) ||
>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>  * @module:		module of creator
>>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>>with
>>>>>  * @parent_refs:	hold references to parent pins pin was registered
>>>>>with
>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>  * @rclk_dev_name:	holds name of device when pin can recover clock
>>>>>from it
>>>>>  * @refcount:		refcount
>>>>>  **/
>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>> 	struct module *module;
>>>>> 	struct xarray dpll_refs;
>>>>> 	struct xarray parent_refs;
>>>>>-	const struct dpll_pin_properties *prop;
>>>>>+	struct dpll_pin_properties prop;
>>>>> 	refcount_t refcount;
>>>>> };
>>>>>
>>>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg, struct
>>>>>dpll_pin *pin,
>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq), &freq,
>>>>> 			  DPLL_A_PIN_PAD))
>>>>> 		return -EMSGSIZE;
>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>> 		nest = nla_nest_start(msg, DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>> 		if (!nest)
>>>>> 			return -EMSGSIZE;
>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN, sizeof(freq),
>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>> 			nla_nest_cancel(msg, nest);
>>>>> 			return -EMSGSIZE;
>>>>> 		}
>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX, sizeof(freq),
>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>> 			nla_nest_cancel(msg, nest);
>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>dpll_pin
>>>>>*pin, u32 freq)
>>>>> {
>>>>> 	int fs;
>>>>>
>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>> 			return true;
>>>>> 	return false;
>>>>> }
>>>>>@@ -403,7 +403,7 @@ static int
>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>> 		     struct netlink_ext_ack *extack)
>>>>> {
>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>> 	struct dpll_pin_ref *ref;
>>>>> 	int ret;
>>>>>
>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin, u32
>>>>>parent_idx,
>>>>> 	int ret;
>>>>>
>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>-	      pin->prop->capabilities)) {
>>>>>+	      pin->prop.capabilities)) {
>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>> 		return -EOPNOTSUPP;
>>>>> 	}
>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll, struct
>>>>>dpll_pin *pin,
>>>>> 	int ret;
>>>>>
>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>-	      pin->prop->capabilities)) {
>>>>>+	      pin->prop.capabilities)) {
>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>> 		return -EOPNOTSUPP;
>>>>> 	}
>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>>>>dpll_pin *pin,
>>>>> 	int ret;
>>>>>
>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>-	      pin->prop->capabilities)) {
>>>>>+	      pin->prop.capabilities)) {
>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>> 		return -EOPNOTSUPP;
>>>>> 	}
>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin, struct
>>>>>dpll_device *dpll,
>>>>> 	int ret;
>>>>>
>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>-	      pin->prop->capabilities)) {
>>>>>+	      pin->prop.capabilities)) {
>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not allowed");
>>>>> 		return -EOPNOTSUPP;
>>>>> 	}
>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct
>>>>>nlattr *phase_adj_attr,
>>>>> 	int ret;
>>>>>
>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>> 				    "phase adjust value not supported");
>>>>> 		return -EINVAL;
>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>*mod_name_attr,
>>>>> 	unsigned long i;
>>>>>
>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>-		prop = pin->prop;
>>>>>+		prop = &pin->prop;
>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>--
>>>>>2.38.1
>>>>>
>>>

Re: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Jiri Pirko]

Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>Sent: Thursday, November 9, 2023 11:51 AM
>>
>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>> Fix issues when performing unordered unbind/bind of a kernel modules
>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>> Currently only serialized bind/unbind of such use case works, fix
>>> the issues and allow for unserialized kernel module bind order.
>>>
>>> The issues are observed on the ice driver, i.e.,
>>>
>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>
>>> results in:
>>>
>>> ice 0000:af:00.0: Removed PTP clock
>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>> PF: supervisor read access in kernel mode
>>> PF: error_code(0x0000) - not-present page
>>> PGD 0 P4D 0
>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-
>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66
>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b 10
>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>knlGS:0000000000000000
>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>> PKRU: 55555554
>>> Call Trace:
>>>   <TASK>
>>>   ? __die+0x20/0x70
>>>   ? page_fault_oops+0x76/0x170
>>>   ? exc_page_fault+0x65/0x150
>>>   ? asm_exc_page_fault+0x22/0x30
>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>   dpll_pin_event_send+0x7d/0x150
>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>   ice_remove+0xcd/0x200 [ice]
>>>   pci_device_remove+0x33/0xa0
>>>   device_release_driver_internal+0x193/0x200
>>>   unbind_store+0x9d/0xb0
>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>   vfs_write+0x2bb/0x3e0
>>>   ksys_write+0x5f/0xe0
>>>   do_syscall_64+0x59/0x90
>>>   ? filp_close+0x1b/0x30
>>>   ? do_dup2+0x7d/0xd0
>>>   ? syscall_exit_work+0x103/0x130
>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>   ? do_syscall_64+0x69/0x90
>>>   ? syscall_exit_work+0x103/0x130
>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>   ? do_syscall_64+0x69/0x90
>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>> RIP: 0033:0x7fdc7d93eb97
>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e
>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0
>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>   </TASK>
>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio
>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c
>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod
>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr
>>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr i2c_i801
>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache jbd2
>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
>>> CR2: 0000000000000010
>>>
>>> Arkadiusz Kubalewski (3):
>>>    dpll: fix pin dump crash after module unbind
>>>    dpll: fix pin dump crash for rebound module
>>>    dpll: fix register pin with unregistered parent pin
>>>
>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++---------------
>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>
>>
>>
>>I still don't get how can we end up with unregistered pin. And shouldn't
>>drivers do unregister of dpll/pin during release procedure? I thought it
>>was kind of agreement we reached while developing the subsystem.
>>
>
>It's definitely not about ending up with unregistered pins.
>
>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in opposite
>order: PF3, PF2, PF1, PF0. And this is working without any issues.

Please fix this in the driver.


>
>Above crash is caused because of unordered driver unload, where dpll subsystem
>tries to notify muxed pin was deleted, but at that time the parent is already
>gone, thus data points to memory which is no longer available, thus crash
>happens when trying to dump pin parents.
>
>This series fixes all issues I could find connected to the situation where
>muxed-pins are trying to access their parents, when parent registerer was removed
>in the meantime.
>
>Thank you!
>Arkadiusz

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 7:04 PM
>
>Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>Sent: Thursday, November 9, 2023 11:56 AM
>>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>>>
>>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>>
>>>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>>> wrote:
>>>>>> In case of multiple kernel module instances using the same dpll
>>>>>>device:
>>>>>> if only one registers dpll device, then only that one can register
>>>>>
>>>>> They why you don't register in multiple instances? See mlx5 for a
>>>>> reference.
>>>>>
>>>>
>>>> Every registration requires ops, but for our case only PF0 is able to
>>>> control dpll pins and device, thus only this can provide ops.
>>>> Basically without PF0, dpll is not able to be controlled, as well
>>>> as directly connected pins.
>>>>
>>>But why do you need other pins then, if FP0 doesn't exist?
>>>
>>
>>In general we don't need them at that point, but this is a corner case,
>>where users for some reason decided to unbind PF 0, and I treat this state
>>as temporary, where dpll/pins controllability is temporarily broken.
>
>So resolve this broken situation internally in the driver, registering
>things only in case PF0 is present. Some simple notification infra would
>do. Don't drag this into the subsystem internals.
>

Thanks for your feedback, but this is already wrong advice.

Our HW/FW is designed in different way than yours, it doesn't mean it is wrong.
As you might recall from our sync meetings, the dpll subsystem is to unify
approaches and reduce the code in the drivers, where your advice is exactly
opposite, suggested fix would require to implement extra synchronization of the
dpll and pin registration state between driver instances, most probably with
use of additional modules like aux-bus or something similar, which was from the
very beginning something we tried to avoid.
Only ice uses the infrastructure of muxed pins, and this is broken as it
doesn't allow unbind the driver which have registered dpll and pins without
crashing the kernel, so a fix is required in dpll subsystem, not in the driver.

Thank you!
Arkadiusz

>
>>
>>The dpll at that point is not registered, all the direct pins are also
>>not registered, thus not available to the users.
>>
>>When I do dump at that point there are still 3 pins present, one for each
>>PF, although they are all zombies - no parents as their parent pins are
>>not
>>registered (as the other patch [1/3] prevents dump of pin parent if the
>>parent is not registered). Maybe we can remove the REGISTERED mark for all
>>the muxed pins, if all their parents have been unregistered, so they won't
>>be visible to the user at all. Will try to POC that.
>>
>>>>>
>>>>>> directly connected pins with a dpll device. If unregistered parent
>>>>>> determines if the muxed pin can be register with it or not, it forces
>>>>>> serialized driver load order - first the driver instance which
>>>>>> registers the direct pins needs to be loaded, then the other
>>>>>> instances
>>>>>> could register muxed type pins.
>>>>>>
>>>>>> Allow registration of a pin with a parent even if the parent was not
>>>>>> yet registered, thus allow ability for unserialized driver instance
>>>>>
>>>>> Weird.
>>>>>
>>>>
>>>> Yeah, this is issue only for MUX/parent pin part, couldn't find better
>>>> way, but it doesn't seem to break things around..
>>>>
>>>
>>>I just wonder how do you see the registration procedure? How can parent
>>>pin exist if it's not registered? I believe you cannot get it through
>>>DPLL API, then the only possible way is to create it within the same
>>>driver code, which can be simply re-arranged. Am I wrong here?
>>>
>>
>>By "parent exist" I mean the parent pin exist in the dpll subsystem
>>(allocated on pins xa), but it doesn't mean it is available to the users,
>>as it might not be registered with a dpll device.
>>
>>We have this 2 step init approach:
>>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>>2.1. dpll_pin_register(..) -> register with a dpll device
>>2.2. dpll_pin_on_pin_register -> register with a parent pin
>>
>>Basically:
>>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for its
>>  recovery clock pin,
>>- other PF's only do step 1 for the direct input pins (as they must get
>>  reference to those in order to register recovery clock pin with them),
>>  and steps: 1 & 2.2 for their recovery clock pin.
>>
>>
>>Thank you!
>>Arkadiusz
>>
>>>> Thank you!
>>>> Arkadiusz
>>>>
>>>>>
>>>>>> load order.
>>>>>> Do not WARN_ON notification for unregistered pin, which can be
>>>>>> invoked
>>>>>> for described case, instead just return error.
>>>>>>
>>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>> functions")
>>>>>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>> ---
>>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>>
>>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>> index
>>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>>> --- a/drivers/dpll/dpll_core.c
>>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>> DPLL_REGISTERED))
>>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>> DPLL_REGISTERED))
>>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id,
>>>>>> DPLL_REGISTERED))
>>>>>>
>>>>>> struct dpll_device_registration {
>>>>>> 	struct list_head list;
>>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>> *parent,
>>>>>> struct dpll_pin *pin,
>>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>>> 	    WARN_ON(!ops->direction_get))
>>>>>> 		return -EINVAL;
>>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>>> -		return -EINVAL;
>>>>>>
>>>>>> 	mutex_lock(&dpll_lock);
>>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops,
>>>>>> priv); diff
>>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>> index
>>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>>>> dpll_pin *pin)
>>>>>> 	int ret = -ENOMEM;
>>>>>> 	void *hdr;
>>>>>>
>>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id,
>>>>>> DPLL_REGISTERED)))
>>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>>> 		return -ENODEV;
>>>>>>
>>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>>> --
>>>>>> 2.38.1
>>>>>>
>>>
>>

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 7:06 PM
>
>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>
>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>
>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>entirely
>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>that
>>>>>>pin), and kernel module is again bound, the pin properties would not
>>>>>>be
>>>>>>updated (the properties are only assigned when memory for the pin is
>>>>>>allocated), prop pointer still points to the kernel module memory of
>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>
>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>crash.
>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>subsystem,
>>>>>>copy the content from the kernel module when pin is allocated, instead
>>>>>>of
>>>>>>using memory of the kernel module.
>>>>>>
>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>functions")
>>>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>>---
>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>
>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>>>module *module,
>>>>>> 		ret = -EINVAL;
>>>>>> 		goto err;
>>>>>> 	}
>>>>>>-	pin->prop = prop;
>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>
>>>>>Odd, you don't care about the pointer within this structure?
>>>>>
>>>>
>>>>Well, true. Need a fix.
>>>>Wondering if copying idea is better than just assigning prop pointer on
>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>
>>>Not sure what do you mean. Examples please.
>>>
>>
>>Sure,
>>
>>Basically this change:
>>
>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>index ae884b92d68c..06b72d5877c3 100644
>>--- a/drivers/dpll/dpll_core.c
>>+++ b/drivers/dpll/dpll_core.c
>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct module
>>*module,
>>                    pos->pin_idx == pin_idx &&
>>                    pos->module == module) {
>>                        ret = pos;
>>+                       pos->prop = prop;
>>                        refcount_inc(&ret->refcount);
>>                        break;
>>                }
>>
>>would replace whole of this patch changes, although seems a bit hacky.
>
>Or event better, as I suggested in the other patch reply, resolve this
>internally in the driver registering things only when they are valid.
>Much better then to hack anything in dpll core.
>

This approach seemed to me hacky, that is why started with coping the
data.
It is not about registering, rather about unregistering on driver
unbind, which brakes things, and currently cannot be recovered in
described case.

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>*parent,
>>>>>>struct dpll_pin *pin,
>>>>>> 	unsigned long i, stop;
>>>>>> 	int ret;
>>>>>>
>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>> 		return -EINVAL;
>>>>>>
>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>  * @module:		module of creator
>>>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>>>with
>>>>>>  * @parent_refs:	hold references to parent pins pin was registered
>>>>>>with
>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>clock
>>>>>>from it
>>>>>>  * @refcount:		refcount
>>>>>>  **/
>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>> 	struct module *module;
>>>>>> 	struct xarray dpll_refs;
>>>>>> 	struct xarray parent_refs;
>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>+	struct dpll_pin_properties prop;
>>>>>> 	refcount_t refcount;
>>>>>> };
>>>>>>
>>>>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>struct
>>>>>>dpll_pin *pin,
>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>&freq,
>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>> 		return -EMSGSIZE;
>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>> 		nest = nla_nest_start(msg,
>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>> 		if (!nest)
>>>>>> 			return -EMSGSIZE;
>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>sizeof(freq),
>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>> 			return -EMSGSIZE;
>>>>>> 		}
>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>sizeof(freq),
>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>dpll_pin
>>>>>>*pin, u32 freq)
>>>>>> {
>>>>>> 	int fs;
>>>>>>
>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>> 			return true;
>>>>>> 	return false;
>>>>>> }
>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>> {
>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>> 	struct dpll_pin_ref *ref;
>>>>>> 	int ret;
>>>>>>
>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin,
>>>>>>u32
>>>>>>parent_idx,
>>>>>> 	int ret;
>>>>>>
>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>-	      pin->prop->capabilities)) {
>>>>>>+	      pin->prop.capabilities)) {
>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>> 		return -EOPNOTSUPP;
>>>>>> 	}
>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>struct
>>>>>>dpll_pin *pin,
>>>>>> 	int ret;
>>>>>>
>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>-	      pin->prop->capabilities)) {
>>>>>>+	      pin->prop.capabilities)) {
>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>> 		return -EOPNOTSUPP;
>>>>>> 	}
>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>>>>>dpll_pin *pin,
>>>>>> 	int ret;
>>>>>>
>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>-	      pin->prop->capabilities)) {
>>>>>>+	      pin->prop.capabilities)) {
>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>> 		return -EOPNOTSUPP;
>>>>>> 	}
>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>struct
>>>>>>dpll_device *dpll,
>>>>>> 	int ret;
>>>>>>
>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>-	      pin->prop->capabilities)) {
>>>>>>+	      pin->prop.capabilities)) {
>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>allowed");
>>>>>> 		return -EOPNOTSUPP;
>>>>>> 	}
>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>struct
>>>>>>nlattr *phase_adj_attr,
>>>>>> 	int ret;
>>>>>>
>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>> 				    "phase adjust value not supported");
>>>>>> 		return -EINVAL;
>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>*mod_name_attr,
>>>>>> 	unsigned long i;
>>>>>>
>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>-		prop = pin->prop;
>>>>>>+		prop = &pin->prop;
>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>--
>>>>>>2.38.1
>>>>>>
>>>>

RE: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Thursday, November 9, 2023 7:07 PM
>
>Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>Sent: Thursday, November 9, 2023 11:51 AM
>>>
>>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>>> Fix issues when performing unordered unbind/bind of a kernel modules
>>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>>> Currently only serialized bind/unbind of such use case works, fix
>>>> the issues and allow for unserialized kernel module bind order.
>>>>
>>>> The issues are observed on the ice driver, i.e.,
>>>>
>>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>>
>>>> results in:
>>>>
>>>> ice 0000:af:00.0: Removed PTP clock
>>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>>> PF: supervisor read access in kernel mode
>>>> PF: error_code(0x0000) - not-present page
>>>> PGD 0 P4D 0
>>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-
>>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66
>>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b
>>>>10
>>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>>>knlGS:0000000000000000
>>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>>> PKRU: 55555554
>>>> Call Trace:
>>>>   <TASK>
>>>>   ? __die+0x20/0x70
>>>>   ? page_fault_oops+0x76/0x170
>>>>   ? exc_page_fault+0x65/0x150
>>>>   ? asm_exc_page_fault+0x22/0x30
>>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>>   dpll_pin_event_send+0x7d/0x150
>>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>>   ice_remove+0xcd/0x200 [ice]
>>>>   pci_device_remove+0x33/0xa0
>>>>   device_release_driver_internal+0x193/0x200
>>>>   unbind_store+0x9d/0xb0
>>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>>   vfs_write+0x2bb/0x3e0
>>>>   ksys_write+0x5f/0xe0
>>>>   do_syscall_64+0x59/0x90
>>>>   ? filp_close+0x1b/0x30
>>>>   ? do_dup2+0x7d/0xd0
>>>>   ? syscall_exit_work+0x103/0x130
>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>   ? do_syscall_64+0x69/0x90
>>>>   ? syscall_exit_work+0x103/0x130
>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>   ? do_syscall_64+0x69/0x90
>>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>>> RIP: 0033:0x7fdc7d93eb97
>>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e
>>>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0
>>>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
>>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>>   </TASK>
>>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio
>>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c
>>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod
>>>>target_core_mod
>>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr
>>>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr
>>>>i2c_i801
>>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache
>>>>jbd2
>>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
>>>> CR2: 0000000000000010
>>>>
>>>> Arkadiusz Kubalewski (3):
>>>>    dpll: fix pin dump crash after module unbind
>>>>    dpll: fix pin dump crash for rebound module
>>>>    dpll: fix register pin with unregistered parent pin
>>>>
>>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++--------------
>>>>-
>>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>>
>>>
>>>
>>>I still don't get how can we end up with unregistered pin. And shouldn't
>>>drivers do unregister of dpll/pin during release procedure? I thought it
>>>was kind of agreement we reached while developing the subsystem.
>>>
>>
>>It's definitely not about ending up with unregistered pins.
>>
>>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in
>>opposite
>>order: PF3, PF2, PF1, PF0. And this is working without any issues.
>
>Please fix this in the driver.
>

Thanks for your feedback, but this is already wrong advice.

Our HW/FW is designed in different way than yours, it doesn't mean it is wrong.
As you might recall from our sync meetings, the dpll subsystem is to unify
approaches and reduce the code in the drivers, where your advice is exactly
opposite, suggested fix would require to implement extra synchronization of the
dpll and pin registration state between driver instances, most probably with
use of additional modules like aux-bus or something similar, which was from the
very beginning something we tried to avoid.
Only ice uses the infrastructure of muxed pins, and this is broken as it
doesn't allow unbind the driver which have registered dpll and pins without
crashing the kernel, so a fix is required in dpll subsystem, not in the driver.

Thank you!
Arkadiusz

>
>>
>>Above crash is caused because of unordered driver unload, where dpll
>>subsystem
>>tries to notify muxed pin was deleted, but at that time the parent is
>>already
>>gone, thus data points to memory which is no longer available, thus crash
>>happens when trying to dump pin parents.
>>
>>This series fixes all issues I could find connected to the situation where
>>muxed-pins are trying to access their parents, when parent registerer was
>>removed
>>in the meantime.
>>
>>Thank you!
>>Arkadiusz

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Thursday, November 9, 2023 7:06 PM
>>
>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>
>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>
>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>>wrote:
>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>entirely
>>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>>that
>>>>>>>pin), and kernel module is again bound, the pin properties would not
>>>>>>>be
>>>>>>>updated (the properties are only assigned when memory for the pin is
>>>>>>>allocated), prop pointer still points to the kernel module memory of
>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>
>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>crash.
>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>subsystem,
>>>>>>>copy the content from the kernel module when pin is allocated, instead
>>>>>>>of
>>>>>>>using memory of the kernel module.
>>>>>>>
>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>functions")
>>>>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>>>---
>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>
>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>>>>module *module,
>>>>>>> 		ret = -EINVAL;
>>>>>>> 		goto err;
>>>>>>> 	}
>>>>>>>-	pin->prop = prop;
>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>
>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>
>>>>>
>>>>>Well, true. Need a fix.
>>>>>Wondering if copying idea is better than just assigning prop pointer on
>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>
>>>>Not sure what do you mean. Examples please.
>>>>
>>>
>>>Sure,
>>>
>>>Basically this change:
>>>
>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>index ae884b92d68c..06b72d5877c3 100644
>>>--- a/drivers/dpll/dpll_core.c
>>>+++ b/drivers/dpll/dpll_core.c
>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct module
>>>*module,
>>>                    pos->pin_idx == pin_idx &&
>>>                    pos->module == module) {
>>>                        ret = pos;
>>>+                       pos->prop = prop;
>>>                        refcount_inc(&ret->refcount);
>>>                        break;
>>>                }
>>>
>>>would replace whole of this patch changes, although seems a bit hacky.
>>
>>Or event better, as I suggested in the other patch reply, resolve this
>>internally in the driver registering things only when they are valid.
>>Much better then to hack anything in dpll core.
>>
>
>This approach seemed to me hacky, that is why started with coping the
>data.
>It is not about registering, rather about unregistering on driver
>unbind, which brakes things, and currently cannot be recovered in
>described case.

Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
related object. Very clean and simple.


>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>>
>>>>>Thank you!
>>>>>Arkadiusz
>>>>>
>>>>>>
>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>*parent,
>>>>>>>struct dpll_pin *pin,
>>>>>>> 	unsigned long i, stop;
>>>>>>> 	int ret;
>>>>>>>
>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>> 		return -EINVAL;
>>>>>>>
>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>  * @module:		module of creator
>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>>>>with
>>>>>>>  * @parent_refs:	hold references to parent pins pin was registered
>>>>>>>with
>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>>clock
>>>>>>>from it
>>>>>>>  * @refcount:		refcount
>>>>>>>  **/
>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>> 	struct module *module;
>>>>>>> 	struct xarray dpll_refs;
>>>>>>> 	struct xarray parent_refs;
>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>> 	refcount_t refcount;
>>>>>>> };
>>>>>>>
>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>struct
>>>>>>>dpll_pin *pin,
>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>>&freq,
>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>> 		return -EMSGSIZE;
>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>> 		if (!nest)
>>>>>>> 			return -EMSGSIZE;
>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>sizeof(freq),
>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>> 			return -EMSGSIZE;
>>>>>>> 		}
>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>sizeof(freq),
>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>>dpll_pin
>>>>>>>*pin, u32 freq)
>>>>>>> {
>>>>>>> 	int fs;
>>>>>>>
>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>> 			return true;
>>>>>>> 	return false;
>>>>>>> }
>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>> {
>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>> 	int ret;
>>>>>>>
>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin,
>>>>>>>u32
>>>>>>>parent_idx,
>>>>>>> 	int ret;
>>>>>>>
>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>> 		return -EOPNOTSUPP;
>>>>>>> 	}
>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>struct
>>>>>>>dpll_pin *pin,
>>>>>>> 	int ret;
>>>>>>>
>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>> 		return -EOPNOTSUPP;
>>>>>>> 	}
>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll, struct
>>>>>>>dpll_pin *pin,
>>>>>>> 	int ret;
>>>>>>>
>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>>> 		return -EOPNOTSUPP;
>>>>>>> 	}
>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>struct
>>>>>>>dpll_device *dpll,
>>>>>>> 	int ret;
>>>>>>>
>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>allowed");
>>>>>>> 		return -EOPNOTSUPP;
>>>>>>> 	}
>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>struct
>>>>>>>nlattr *phase_adj_attr,
>>>>>>> 	int ret;
>>>>>>>
>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>> 				    "phase adjust value not supported");
>>>>>>> 		return -EINVAL;
>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>*mod_name_attr,
>>>>>>> 	unsigned long i;
>>>>>>>
>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>>-		prop = pin->prop;
>>>>>>>+		prop = &pin->prop;
>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>--
>>>>>>>2.38.1
>>>>>>>
>>>>>
>

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Jiri Pirko]

Fri, Nov 10, 2023 at 12:21:11AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Thursday, November 9, 2023 7:04 PM
>>
>>Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>Sent: Thursday, November 9, 2023 11:56 AM
>>>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>>>>
>>>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>>>
>>>>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>>>> wrote:
>>>>>>> In case of multiple kernel module instances using the same dpll
>>>>>>>device:
>>>>>>> if only one registers dpll device, then only that one can register
>>>>>>
>>>>>> They why you don't register in multiple instances? See mlx5 for a
>>>>>> reference.
>>>>>>
>>>>>
>>>>> Every registration requires ops, but for our case only PF0 is able to
>>>>> control dpll pins and device, thus only this can provide ops.
>>>>> Basically without PF0, dpll is not able to be controlled, as well
>>>>> as directly connected pins.
>>>>>
>>>>But why do you need other pins then, if FP0 doesn't exist?
>>>>
>>>
>>>In general we don't need them at that point, but this is a corner case,
>>>where users for some reason decided to unbind PF 0, and I treat this state
>>>as temporary, where dpll/pins controllability is temporarily broken.
>>
>>So resolve this broken situation internally in the driver, registering
>>things only in case PF0 is present. Some simple notification infra would
>>do. Don't drag this into the subsystem internals.
>>
>
>Thanks for your feedback, but this is already wrong advice.
>
>Our HW/FW is designed in different way than yours, it doesn't mean it is wrong.
>As you might recall from our sync meetings, the dpll subsystem is to unify
>approaches and reduce the code in the drivers, where your advice is exactly

No. Your driver knows when what objects are valid or not. Of a pin of
PF1 is not valid because the "master" PF0 is gone, it is responsibility
of your driver to resolve that. Don't bring this internal dependencies
to the dpll core please, does not make any sense to do so. Thanks!


>opposite, suggested fix would require to implement extra synchronization of the
>dpll and pin registration state between driver instances, most probably with
>use of additional modules like aux-bus or something similar, which was from the
>very beginning something we tried to avoid.
>Only ice uses the infrastructure of muxed pins, and this is broken as it
>doesn't allow unbind the driver which have registered dpll and pins without
>crashing the kernel, so a fix is required in dpll subsystem, not in the driver.
>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>The dpll at that point is not registered, all the direct pins are also
>>>not registered, thus not available to the users.
>>>
>>>When I do dump at that point there are still 3 pins present, one for each
>>>PF, although they are all zombies - no parents as their parent pins are
>>>not
>>>registered (as the other patch [1/3] prevents dump of pin parent if the
>>>parent is not registered). Maybe we can remove the REGISTERED mark for all
>>>the muxed pins, if all their parents have been unregistered, so they won't
>>>be visible to the user at all. Will try to POC that.
>>>
>>>>>>
>>>>>>> directly connected pins with a dpll device. If unregistered parent
>>>>>>> determines if the muxed pin can be register with it or not, it forces
>>>>>>> serialized driver load order - first the driver instance which
>>>>>>> registers the direct pins needs to be loaded, then the other
>>>>>>> instances
>>>>>>> could register muxed type pins.
>>>>>>>
>>>>>>> Allow registration of a pin with a parent even if the parent was not
>>>>>>> yet registered, thus allow ability for unserialized driver instance
>>>>>>
>>>>>> Weird.
>>>>>>
>>>>>
>>>>> Yeah, this is issue only for MUX/parent pin part, couldn't find better
>>>>> way, but it doesn't seem to break things around..
>>>>>
>>>>
>>>>I just wonder how do you see the registration procedure? How can parent
>>>>pin exist if it's not registered? I believe you cannot get it through
>>>>DPLL API, then the only possible way is to create it within the same
>>>>driver code, which can be simply re-arranged. Am I wrong here?
>>>>
>>>
>>>By "parent exist" I mean the parent pin exist in the dpll subsystem
>>>(allocated on pins xa), but it doesn't mean it is available to the users,
>>>as it might not be registered with a dpll device.
>>>
>>>We have this 2 step init approach:
>>>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>>>2.1. dpll_pin_register(..) -> register with a dpll device
>>>2.2. dpll_pin_on_pin_register -> register with a parent pin
>>>
>>>Basically:
>>>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for its
>>>  recovery clock pin,
>>>- other PF's only do step 1 for the direct input pins (as they must get
>>>  reference to those in order to register recovery clock pin with them),
>>>  and steps: 1 & 2.2 for their recovery clock pin.
>>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>> Thank you!
>>>>> Arkadiusz
>>>>>
>>>>>>
>>>>>>> load order.
>>>>>>> Do not WARN_ON notification for unregistered pin, which can be
>>>>>>> invoked
>>>>>>> for described case, instead just return error.
>>>>>>>
>>>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base functions")
>>>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>> functions")
>>>>>>> Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>>> ---
>>>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>>>
>>>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>> index
>>>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>>>> --- a/drivers/dpll/dpll_core.c
>>>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>> DPLL_REGISTERED))
>>>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>> DPLL_REGISTERED))
>>>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id,
>>>>>>> DPLL_REGISTERED))
>>>>>>>
>>>>>>> struct dpll_device_registration {
>>>>>>> 	struct list_head list;
>>>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>> *parent,
>>>>>>> struct dpll_pin *pin,
>>>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>>>> 	    WARN_ON(!ops->direction_get))
>>>>>>> 		return -EINVAL;
>>>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>>>> -		return -EINVAL;
>>>>>>>
>>>>>>> 	mutex_lock(&dpll_lock);
>>>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops,
>>>>>>> priv); diff
>>>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>> index
>>>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>>>>> dpll_pin *pin)
>>>>>>> 	int ret = -ENOMEM;
>>>>>>> 	void *hdr;
>>>>>>>
>>>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id,
>>>>>>> DPLL_REGISTERED)))
>>>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>>>> 		return -ENODEV;
>>>>>>>
>>>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>>>> --
>>>>>>> 2.38.1
>>>>>>>
>>>>
>>>

Re: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Jiri Pirko]

Fri, Nov 10, 2023 at 12:35:43AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Thursday, November 9, 2023 7:07 PM
>>
>>Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>Sent: Thursday, November 9, 2023 11:51 AM
>>>>
>>>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>>>> Fix issues when performing unordered unbind/bind of a kernel modules
>>>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>>>> Currently only serialized bind/unbind of such use case works, fix
>>>>> the issues and allow for unserialized kernel module bind order.
>>>>>
>>>>> The issues are observed on the ice driver, i.e.,
>>>>>
>>>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>>>
>>>>> results in:
>>>>>
>>>>> ice 0000:af:00.0: Removed PTP clock
>>>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>>>> PF: supervisor read access in kernel mode
>>>>> PF: error_code(0x0000) - not-present page
>>>>> PGD 0 P4D 0
>>>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-rc5_next-
>>>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b 66
>>>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b
>>>>>10
>>>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>>>>knlGS:0000000000000000
>>>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>>>> PKRU: 55555554
>>>>> Call Trace:
>>>>>   <TASK>
>>>>>   ? __die+0x20/0x70
>>>>>   ? page_fault_oops+0x76/0x170
>>>>>   ? exc_page_fault+0x65/0x150
>>>>>   ? asm_exc_page_fault+0x22/0x30
>>>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>>>   dpll_pin_event_send+0x7d/0x150
>>>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>>>   ice_remove+0xcd/0x200 [ice]
>>>>>   pci_device_remove+0x33/0xa0
>>>>>   device_release_driver_internal+0x193/0x200
>>>>>   unbind_store+0x9d/0xb0
>>>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>>>   vfs_write+0x2bb/0x3e0
>>>>>   ksys_write+0x5f/0xe0
>>>>>   do_syscall_64+0x59/0x90
>>>>>   ? filp_close+0x1b/0x30
>>>>>   ? do_dup2+0x7d/0xd0
>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>   ? do_syscall_64+0x69/0x90
>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>   ? do_syscall_64+0x69/0x90
>>>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>>>> RIP: 0033:0x7fdc7d93eb97
>>>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e
>>>>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0
>>>>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
>>>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>>>   </TASK>
>>>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1 vfio
>>>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs libcrc32c
>>>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod
>>>>>target_core_mod
>>>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm intel_rapl_msr
>>>>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr
>>>>>i2c_i801
>>>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache
>>>>>jbd2
>>>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded: iavf]
>>>>> CR2: 0000000000000010
>>>>>
>>>>> Arkadiusz Kubalewski (3):
>>>>>    dpll: fix pin dump crash after module unbind
>>>>>    dpll: fix pin dump crash for rebound module
>>>>>    dpll: fix register pin with unregistered parent pin
>>>>>
>>>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++--------------
>>>>>-
>>>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>>>
>>>>
>>>>
>>>>I still don't get how can we end up with unregistered pin. And shouldn't
>>>>drivers do unregister of dpll/pin during release procedure? I thought it
>>>>was kind of agreement we reached while developing the subsystem.
>>>>
>>>
>>>It's definitely not about ending up with unregistered pins.
>>>
>>>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in
>>>opposite
>>>order: PF3, PF2, PF1, PF0. And this is working without any issues.
>>
>>Please fix this in the driver.
>>
>
>Thanks for your feedback, but this is already wrong advice.
>
>Our HW/FW is designed in different way than yours, it doesn't mean it is wrong.
>As you might recall from our sync meetings, the dpll subsystem is to unify
>approaches and reduce the code in the drivers, where your advice is exactly
>opposite, suggested fix would require to implement extra synchronization of the
>dpll and pin registration state between driver instances, most probably with
>use of additional modules like aux-bus or something similar, which was from the
>very beginning something we tried to avoid.
>Only ice uses the infrastructure of muxed pins, and this is broken as it
>doesn't allow unbind the driver which have registered dpll and pins without
>crashing the kernel, so a fix is required in dpll subsystem, not in the driver.

I replied in the other patch thread.


>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Above crash is caused because of unordered driver unload, where dpll
>>>subsystem
>>>tries to notify muxed pin was deleted, but at that time the parent is
>>>already
>>>gone, thus data points to memory which is no longer available, thus crash
>>>happens when trying to dump pin parents.
>>>
>>>This series fixes all issues I could find connected to the situation where
>>>muxed-pins are trying to access their parents, when parent registerer was
>>>removed
>>>in the meantime.
>>>
>>>Thank you!
>>>Arkadiusz

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 7:48 AM
>
>Fri, Nov 10, 2023 at 12:21:11AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Thursday, November 9, 2023 7:04 PM
>>>
>>>Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>Sent: Thursday, November 9, 2023 11:56 AM
>>>>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>>>>>
>>>>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>>>>
>>>>>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>>>>> wrote:
>>>>>>>> In case of multiple kernel module instances using the same dpll
>>>>>>>>device:
>>>>>>>> if only one registers dpll device, then only that one can register
>>>>>>>
>>>>>>> They why you don't register in multiple instances? See mlx5 for a
>>>>>>> reference.
>>>>>>>
>>>>>>
>>>>>> Every registration requires ops, but for our case only PF0 is able to
>>>>>> control dpll pins and device, thus only this can provide ops.
>>>>>> Basically without PF0, dpll is not able to be controlled, as well
>>>>>> as directly connected pins.
>>>>>>
>>>>>But why do you need other pins then, if FP0 doesn't exist?
>>>>>
>>>>
>>>>In general we don't need them at that point, but this is a corner case,
>>>>where users for some reason decided to unbind PF 0, and I treat this
>>>>state
>>>>as temporary, where dpll/pins controllability is temporarily broken.
>>>
>>>So resolve this broken situation internally in the driver, registering
>>>things only in case PF0 is present. Some simple notification infra would
>>>do. Don't drag this into the subsystem internals.
>>>
>>
>>Thanks for your feedback, but this is already wrong advice.
>>
>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>wrong.
>>As you might recall from our sync meetings, the dpll subsystem is to unify
>>approaches and reduce the code in the drivers, where your advice is
>>exactly
>
>No. Your driver knows when what objects are valid or not. Of a pin of
>PF1 is not valid because the "master" PF0 is gone, it is responsibility
>of your driver to resolve that. Don't bring this internal dependencies
>to the dpll core please, does not make any sense to do so. Thanks!
>

No, a driver doesn't know it, those are separated instances, and you already
suggested to implement special notification bus in the driver.
This is not needed and prone for another errors. The dpll subsystem is here to
make driver life easier.

Thank you!
Arkadiusz

>
>>opposite, suggested fix would require to implement extra synchronization
>>of the
>>dpll and pin registration state between driver instances, most probably
>>with
>>use of additional modules like aux-bus or something similar, which was
>>from the
>>very beginning something we tried to avoid.
>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>doesn't allow unbind the driver which have registered dpll and pins
>>without
>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>driver.
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>The dpll at that point is not registered, all the direct pins are also
>>>>not registered, thus not available to the users.
>>>>
>>>>When I do dump at that point there are still 3 pins present, one for
>>>>each
>>>>PF, although they are all zombies - no parents as their parent pins are
>>>>not
>>>>registered (as the other patch [1/3] prevents dump of pin parent if the
>>>>parent is not registered). Maybe we can remove the REGISTERED mark for
>>>>all
>>>>the muxed pins, if all their parents have been unregistered, so they
>>>>won't
>>>>be visible to the user at all. Will try to POC that.
>>>>
>>>>>>>
>>>>>>>> directly connected pins with a dpll device. If unregistered parent
>>>>>>>> determines if the muxed pin can be register with it or not, it
>>>>>>>>forces
>>>>>>>> serialized driver load order - first the driver instance which
>>>>>>>> registers the direct pins needs to be loaded, then the other
>>>>>>>> instances
>>>>>>>> could register muxed type pins.
>>>>>>>>
>>>>>>>> Allow registration of a pin with a parent even if the parent was
>>>>>>>>not
>>>>>>>> yet registered, thus allow ability for unserialized driver instance
>>>>>>>
>>>>>>> Weird.
>>>>>>>
>>>>>>
>>>>>> Yeah, this is issue only for MUX/parent pin part, couldn't find
>>>>>>better
>>>>>> way, but it doesn't seem to break things around..
>>>>>>
>>>>>
>>>>>I just wonder how do you see the registration procedure? How can parent
>>>>>pin exist if it's not registered? I believe you cannot get it through
>>>>>DPLL API, then the only possible way is to create it within the same
>>>>>driver code, which can be simply re-arranged. Am I wrong here?
>>>>>
>>>>
>>>>By "parent exist" I mean the parent pin exist in the dpll subsystem
>>>>(allocated on pins xa), but it doesn't mean it is available to the
>>>>users,
>>>>as it might not be registered with a dpll device.
>>>>
>>>>We have this 2 step init approach:
>>>>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>>>>2.1. dpll_pin_register(..) -> register with a dpll device
>>>>2.2. dpll_pin_on_pin_register -> register with a parent pin
>>>>
>>>>Basically:
>>>>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for
>>>>its
>>>>  recovery clock pin,
>>>>- other PF's only do step 1 for the direct input pins (as they must get
>>>>  reference to those in order to register recovery clock pin with them),
>>>>  and steps: 1 & 2.2 for their recovery clock pin.
>>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>> Thank you!
>>>>>> Arkadiusz
>>>>>>
>>>>>>>
>>>>>>>> load order.
>>>>>>>> Do not WARN_ON notification for unregistered pin, which can be
>>>>>>>> invoked
>>>>>>>> for described case, instead just return error.
>>>>>>>>
>>>>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>functions")
>>>>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>> functions")
>>>>>>>> Signed-off-by: Arkadiusz Kubalewski
>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>> ---
>>>>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>> index
>>>>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>>>>> --- a/drivers/dpll/dpll_core.c
>>>>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>> DPLL_REGISTERED))
>>>>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>> DPLL_REGISTERED))
>>>>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id,
>>>>>>>> DPLL_REGISTERED))
>>>>>>>>
>>>>>>>> struct dpll_device_registration {
>>>>>>>> 	struct list_head list;
>>>>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>> *parent,
>>>>>>>> struct dpll_pin *pin,
>>>>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>>>>> 	    WARN_ON(!ops->direction_get))
>>>>>>>> 		return -EINVAL;
>>>>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>>>>> -		return -EINVAL;
>>>>>>>>
>>>>>>>> 	mutex_lock(&dpll_lock);
>>>>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops,
>>>>>>>> priv); diff
>>>>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>>> index
>>>>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>>>>>> dpll_pin *pin)
>>>>>>>> 	int ret = -ENOMEM;
>>>>>>>> 	void *hdr;
>>>>>>>>
>>>>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id,
>>>>>>>> DPLL_REGISTERED)))
>>>>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>>>>> 		return -ENODEV;
>>>>>>>>
>>>>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>>>>> --
>>>>>>>> 2.38.1
>>>>>>>>
>>>>>
>>>>

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 7:46 AM
>
>Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Thursday, November 9, 2023 7:06 PM
>>>
>>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>>
>>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>>
>>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>>>wrote:
>>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>>entirely
>>>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>>>that
>>>>>>>>pin), and kernel module is again bound, the pin properties would not
>>>>>>>>be
>>>>>>>>updated (the properties are only assigned when memory for the pin is
>>>>>>>>allocated), prop pointer still points to the kernel module memory of
>>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>>
>>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>>crash.
>>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>>subsystem,
>>>>>>>>copy the content from the kernel module when pin is allocated,
>>>>>>>>instead
>>>>>>>>of
>>>>>>>>using memory of the kernel module.
>>>>>>>>
>>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>functions")
>>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>functions")
>>>>>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>>>>---
>>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>>
>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>>>>>module *module,
>>>>>>>> 		ret = -EINVAL;
>>>>>>>> 		goto err;
>>>>>>>> 	}
>>>>>>>>-	pin->prop = prop;
>>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>>
>>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>>
>>>>>>
>>>>>>Well, true. Need a fix.
>>>>>>Wondering if copying idea is better than just assigning prop pointer
>>>>>>on
>>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>>
>>>>>Not sure what do you mean. Examples please.
>>>>>
>>>>
>>>>Sure,
>>>>
>>>>Basically this change:
>>>>
>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>index ae884b92d68c..06b72d5877c3 100644
>>>>--- a/drivers/dpll/dpll_core.c
>>>>+++ b/drivers/dpll/dpll_core.c
>>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct
>>>>module
>>>>*module,
>>>>                    pos->pin_idx == pin_idx &&
>>>>                    pos->module == module) {
>>>>                        ret = pos;
>>>>+                       pos->prop = prop;
>>>>                        refcount_inc(&ret->refcount);
>>>>                        break;
>>>>                }
>>>>
>>>>would replace whole of this patch changes, although seems a bit hacky.
>>>
>>>Or event better, as I suggested in the other patch reply, resolve this
>>>internally in the driver registering things only when they are valid.
>>>Much better then to hack anything in dpll core.
>>>
>>
>>This approach seemed to me hacky, that is why started with coping the
>>data.
>>It is not about registering, rather about unregistering on driver
>>unbind, which brakes things, and currently cannot be recovered in
>>described case.
>
>Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
>related object. Very clean and simple.
>

What you are suggesting is:
- special purpose bus in the driver,
- dpll-related,
- not needed,
- prone for errors.

The dpll subsystem is here to make driver life easier.

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>>
>>>>>>Thank you!
>>>>>>Arkadiusz
>>>>>>
>>>>>>>
>>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>*parent,
>>>>>>>>struct dpll_pin *pin,
>>>>>>>> 	unsigned long i, stop;
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>>> 		return -EINVAL;
>>>>>>>>
>>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>>  * @module:		module of creator
>>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>>>>>with
>>>>>>>>  * @parent_refs:	hold references to parent pins pin was
>registered
>>>>>>>>with
>>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>>>clock
>>>>>>>>from it
>>>>>>>>  * @refcount:		refcount
>>>>>>>>  **/
>>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>>> 	struct module *module;
>>>>>>>> 	struct xarray dpll_refs;
>>>>>>>> 	struct xarray parent_refs;
>>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>>> 	refcount_t refcount;
>>>>>>>> };
>>>>>>>>
>>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c
>b/drivers/dpll/dpll_netlink.c
>>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>>struct
>>>>>>>>dpll_pin *pin,
>>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>>>&freq,
>>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>>> 		return -EMSGSIZE;
>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>>> 		if (!nest)
>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>>sizeof(freq),
>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>> 			return -EMSGSIZE;
>>>>>>>> 		}
>>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>>sizeof(freq),
>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>>>dpll_pin
>>>>>>>>*pin, u32 freq)
>>>>>>>> {
>>>>>>>> 	int fs;
>>>>>>>>
>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>>> 			return true;
>>>>>>>> 	return false;
>>>>>>>> }
>>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>>> {
>>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin,
>>>>>>>>u32
>>>>>>>>parent_idx,
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>> 	}
>>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>>struct
>>>>>>>>dpll_pin *pin,
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>> 	}
>>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll,
>struct
>>>>>>>>dpll_pin *pin,
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>> 	}
>>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>>struct
>>>>>>>>dpll_device *dpll,
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>>allowed");
>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>> 	}
>>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>>struct
>>>>>>>>nlattr *phase_adj_attr,
>>>>>>>> 	int ret;
>>>>>>>>
>>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>>> 				    "phase adjust value not supported");
>>>>>>>> 		return -EINVAL;
>>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>>*mod_name_attr,
>>>>>>>> 	unsigned long i;
>>>>>>>>
>>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>>>-		prop = pin->prop;
>>>>>>>>+		prop = &pin->prop;
>>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>>--
>>>>>>>>2.38.1
>>>>>>>>
>>>>>>
>>

RE: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 7:49 AM
>
>Fri, Nov 10, 2023 at 12:35:43AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Thursday, November 9, 2023 7:07 PM
>>>
>>>Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>Sent: Thursday, November 9, 2023 11:51 AM
>>>>>
>>>>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>>>>> Fix issues when performing unordered unbind/bind of a kernel modules
>>>>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>>>>> Currently only serialized bind/unbind of such use case works, fix
>>>>>> the issues and allow for unserialized kernel module bind order.
>>>>>>
>>>>>> The issues are observed on the ice driver, i.e.,
>>>>>>
>>>>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>>>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>>>>
>>>>>> results in:
>>>>>>
>>>>>> ice 0000:af:00.0: Removed PTP clock
>>>>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>>>>> PF: supervisor read access in kernel mode
>>>>>> PF: error_code(0x0000) - not-present page
>>>>>> PGD 0 P4D 0
>>>>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>>>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-
>>>>>>rc5_next-
>>>>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>>>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>>>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b
>>>>>>66
>>>>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b
>>>>>>10
>>>>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>>>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>>>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>>>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>>>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>>>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>>>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>>>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>>>>>knlGS:0000000000000000
>>>>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>>>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>>>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>>>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>>>>> PKRU: 55555554
>>>>>> Call Trace:
>>>>>>   <TASK>
>>>>>>   ? __die+0x20/0x70
>>>>>>   ? page_fault_oops+0x76/0x170
>>>>>>   ? exc_page_fault+0x65/0x150
>>>>>>   ? asm_exc_page_fault+0x22/0x30
>>>>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>>>>   dpll_pin_event_send+0x7d/0x150
>>>>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>>>>   ice_remove+0xcd/0x200 [ice]
>>>>>>   pci_device_remove+0x33/0xa0
>>>>>>   device_release_driver_internal+0x193/0x200
>>>>>>   unbind_store+0x9d/0xb0
>>>>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>>>>   vfs_write+0x2bb/0x3e0
>>>>>>   ksys_write+0x5f/0xe0
>>>>>>   do_syscall_64+0x59/0x90
>>>>>>   ? filp_close+0x1b/0x30
>>>>>>   ? do_dup2+0x7d/0xd0
>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>>>>> RIP: 0033:0x7fdc7d93eb97
>>>>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f
>>>>>>1e
>>>>>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00
>>>>>>f0
>>>>>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>>>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX:
>>>>>>0000000000000001
>>>>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>>>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>>>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>>>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>>>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>>>>   </TASK>
>>>>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1
>>>>>>vfio
>>>>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs
>>>>>>libcrc32c
>>>>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod
>>>>>>target_core_mod
>>>>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm
>>>>>>intel_rapl_msr
>>>>>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>>>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>>>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr
>>>>>>i2c_i801
>>>>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache
>>>>>>jbd2
>>>>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>>>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>>>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded:
>>>>>>iavf]
>>>>>> CR2: 0000000000000010
>>>>>>
>>>>>> Arkadiusz Kubalewski (3):
>>>>>>    dpll: fix pin dump crash after module unbind
>>>>>>    dpll: fix pin dump crash for rebound module
>>>>>>    dpll: fix register pin with unregistered parent pin
>>>>>>
>>>>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++------------
>>>>>>--
>>>>>>-
>>>>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>>>>
>>>>>
>>>>>
>>>>>I still don't get how can we end up with unregistered pin. And
>>>>>shouldn't
>>>>>drivers do unregister of dpll/pin during release procedure? I thought
>>>>>it
>>>>>was kind of agreement we reached while developing the subsystem.
>>>>>
>>>>
>>>>It's definitely not about ending up with unregistered pins.
>>>>
>>>>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in
>>>>opposite
>>>>order: PF3, PF2, PF1, PF0. And this is working without any issues.
>>>
>>>Please fix this in the driver.
>>>
>>
>>Thanks for your feedback, but this is already wrong advice.
>>
>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>wrong.
>>As you might recall from our sync meetings, the dpll subsystem is to unify
>>approaches and reduce the code in the drivers, where your advice is
>>exactly
>>opposite, suggested fix would require to implement extra synchronization
>>of the
>>dpll and pin registration state between driver instances, most probably
>>with
>>use of additional modules like aux-bus or something similar, which was
>>from the
>>very beginning something we tried to avoid.
>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>doesn't allow unbind the driver which have registered dpll and pins
>>without
>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>driver.
>
>I replied in the other patch thread.
>

Yes, so did I.
But what is the reason you have moved the discussion from the other thread
into this one?

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Above crash is caused because of unordered driver unload, where dpll
>>>>subsystem
>>>>tries to notify muxed pin was deleted, but at that time the parent is
>>>>already
>>>>gone, thus data points to memory which is no longer available, thus
>>>>crash
>>>>happens when trying to dump pin parents.
>>>>
>>>>This series fixes all issues I could find connected to the situation
>>>>where
>>>>muxed-pins are trying to access their parents, when parent registerer
>>>>was
>>>>removed
>>>>in the meantime.
>>>>
>>>>Thank you!
>>>>Arkadiusz

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Fri, Nov 10, 2023 at 10:01:50AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Friday, November 10, 2023 7:46 AM
>>
>>Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Thursday, November 9, 2023 7:06 PM
>>>>
>>>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>>>
>>>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>>wrote:
>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>>>
>>>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>>>>wrote:
>>>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>>>entirely
>>>>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>>>>that
>>>>>>>>>pin), and kernel module is again bound, the pin properties would not
>>>>>>>>>be
>>>>>>>>>updated (the properties are only assigned when memory for the pin is
>>>>>>>>>allocated), prop pointer still points to the kernel module memory of
>>>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>>>
>>>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>>>crash.
>>>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>>>subsystem,
>>>>>>>>>copy the content from the kernel module when pin is allocated,
>>>>>>>>>instead
>>>>>>>>>of
>>>>>>>>>using memory of the kernel module.
>>>>>>>>>
>>>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>functions")
>>>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>>functions")
>>>>>>>>>Signed-off-by: Arkadiusz Kubalewski <arkadiusz.kubalewski@intel.com>
>>>>>>>>>---
>>>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>>>
>>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct
>>>>>>>>>module *module,
>>>>>>>>> 		ret = -EINVAL;
>>>>>>>>> 		goto err;
>>>>>>>>> 	}
>>>>>>>>>-	pin->prop = prop;
>>>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>>>
>>>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>>>
>>>>>>>
>>>>>>>Well, true. Need a fix.
>>>>>>>Wondering if copying idea is better than just assigning prop pointer
>>>>>>>on
>>>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>>>
>>>>>>Not sure what do you mean. Examples please.
>>>>>>
>>>>>
>>>>>Sure,
>>>>>
>>>>>Basically this change:
>>>>>
>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>index ae884b92d68c..06b72d5877c3 100644
>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct
>>>>>module
>>>>>*module,
>>>>>                    pos->pin_idx == pin_idx &&
>>>>>                    pos->module == module) {
>>>>>                        ret = pos;
>>>>>+                       pos->prop = prop;
>>>>>                        refcount_inc(&ret->refcount);
>>>>>                        break;
>>>>>                }
>>>>>
>>>>>would replace whole of this patch changes, although seems a bit hacky.
>>>>
>>>>Or event better, as I suggested in the other patch reply, resolve this
>>>>internally in the driver registering things only when they are valid.
>>>>Much better then to hack anything in dpll core.
>>>>
>>>
>>>This approach seemed to me hacky, that is why started with coping the
>>>data.
>>>It is not about registering, rather about unregistering on driver
>>>unbind, which brakes things, and currently cannot be recovered in
>>>described case.
>>
>>Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
>>related object. Very clean and simple.
>>
>
>What you are suggesting is:
>- special purpose bus in the driver,

No, it is a simple notificator. Very common infra over the whole
kernel code.


>- dpll-related,

Is this the only thing that PF0 is special with? Perhaps you can
utilize this for other features as well, since your fw design is like
this.


>- not needed,
>- prone for errors.
>
>The dpll subsystem is here to make driver life easier.

No, the subsystem is never here to handle device specific issues. And
your PF0 dependency is very clearly something device specific. Don't
pollute the dpll subsystem with workaround to handle specific device
needs. create/register the dplls objects from your driver only when it
is valid to do so. Make sure the lifetime of such object stays in
the scope of validity. Handle that in the driver. Very clear and simple.

Thanks!


>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>>
>>>>>Thank you!
>>>>>Arkadiusz
>>>>>
>>>>>>
>>>>>>>
>>>>>>>Thank you!
>>>>>>>Arkadiusz
>>>>>>>
>>>>>>>>
>>>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>>*parent,
>>>>>>>>>struct dpll_pin *pin,
>>>>>>>>> 	unsigned long i, stop;
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>>>> 		return -EINVAL;
>>>>>>>>>
>>>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>>>  * @module:		module of creator
>>>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was registered
>>>>>>>>>with
>>>>>>>>>  * @parent_refs:	hold references to parent pins pin was
>>registered
>>>>>>>>>with
>>>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>>>>clock
>>>>>>>>>from it
>>>>>>>>>  * @refcount:		refcount
>>>>>>>>>  **/
>>>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>>>> 	struct module *module;
>>>>>>>>> 	struct xarray dpll_refs;
>>>>>>>>> 	struct xarray parent_refs;
>>>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>>>> 	refcount_t refcount;
>>>>>>>>> };
>>>>>>>>>
>>>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c
>>b/drivers/dpll/dpll_netlink.c
>>>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>>>struct
>>>>>>>>>dpll_pin *pin,
>>>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>>>>&freq,
>>>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>>>> 		return -EMSGSIZE;
>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>>>> 		if (!nest)
>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>>>sizeof(freq),
>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>> 		}
>>>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>>>sizeof(freq),
>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>>>>dpll_pin
>>>>>>>>>*pin, u32 freq)
>>>>>>>>> {
>>>>>>>>> 	int fs;
>>>>>>>>>
>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>>>> 			return true;
>>>>>>>>> 	return false;
>>>>>>>>> }
>>>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>>>> {
>>>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin *pin,
>>>>>>>>>u32
>>>>>>>>>parent_idx,
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>> 	}
>>>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>>>struct
>>>>>>>>>dpll_pin *pin,
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>> 	}
>>>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll,
>>struct
>>>>>>>>>dpll_pin *pin,
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>> 	}
>>>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>>>struct
>>>>>>>>>dpll_device *dpll,
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>>>allowed");
>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>> 	}
>>>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>>>struct
>>>>>>>>>nlattr *phase_adj_attr,
>>>>>>>>> 	int ret;
>>>>>>>>>
>>>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>>>> 				    "phase adjust value not supported");
>>>>>>>>> 		return -EINVAL;
>>>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>>>*mod_name_attr,
>>>>>>>>> 	unsigned long i;
>>>>>>>>>
>>>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>>>>-		prop = pin->prop;
>>>>>>>>>+		prop = &pin->prop;
>>>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>>>--
>>>>>>>>>2.38.1
>>>>>>>>>
>>>>>>>
>>>

Re: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Jiri Pirko]

Fri, Nov 10, 2023 at 09:50:34AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Friday, November 10, 2023 7:48 AM
>>
>>Fri, Nov 10, 2023 at 12:21:11AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Thursday, November 9, 2023 7:04 PM
>>>>
>>>>Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>>Sent: Thursday, November 9, 2023 11:56 AM
>>>>>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri Pirko
>>>>>>
>>>>>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>>>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>>>>>
>>>>>>>> Wed, Nov 08, 2023 at 11:32:26AM CET, arkadiusz.kubalewski@intel.com
>>>>>>>> wrote:
>>>>>>>>> In case of multiple kernel module instances using the same dpll
>>>>>>>>>device:
>>>>>>>>> if only one registers dpll device, then only that one can register
>>>>>>>>
>>>>>>>> They why you don't register in multiple instances? See mlx5 for a
>>>>>>>> reference.
>>>>>>>>
>>>>>>>
>>>>>>> Every registration requires ops, but for our case only PF0 is able to
>>>>>>> control dpll pins and device, thus only this can provide ops.
>>>>>>> Basically without PF0, dpll is not able to be controlled, as well
>>>>>>> as directly connected pins.
>>>>>>>
>>>>>>But why do you need other pins then, if FP0 doesn't exist?
>>>>>>
>>>>>
>>>>>In general we don't need them at that point, but this is a corner case,
>>>>>where users for some reason decided to unbind PF 0, and I treat this
>>>>>state
>>>>>as temporary, where dpll/pins controllability is temporarily broken.
>>>>
>>>>So resolve this broken situation internally in the driver, registering
>>>>things only in case PF0 is present. Some simple notification infra would
>>>>do. Don't drag this into the subsystem internals.
>>>>
>>>
>>>Thanks for your feedback, but this is already wrong advice.
>>>
>>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>>wrong.
>>>As you might recall from our sync meetings, the dpll subsystem is to unify
>>>approaches and reduce the code in the drivers, where your advice is
>>>exactly
>>
>>No. Your driver knows when what objects are valid or not. Of a pin of
>>PF1 is not valid because the "master" PF0 is gone, it is responsibility
>>of your driver to resolve that. Don't bring this internal dependencies
>>to the dpll core please, does not make any sense to do so. Thanks!
>>
>
>No, a driver doesn't know it, those are separated instances, and you already
>suggested to implement special notification bus in the driver.
>This is not needed and prone for another errors. The dpll subsystem is here to
>make driver life easier.

See the other thread for my reply.

>
>Thank you!
>Arkadiusz
>
>>
>>>opposite, suggested fix would require to implement extra synchronization
>>>of the
>>>dpll and pin registration state between driver instances, most probably
>>>with
>>>use of additional modules like aux-bus or something similar, which was
>>>from the
>>>very beginning something we tried to avoid.
>>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>>doesn't allow unbind the driver which have registered dpll and pins
>>>without
>>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>>driver.
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>>
>>>>>The dpll at that point is not registered, all the direct pins are also
>>>>>not registered, thus not available to the users.
>>>>>
>>>>>When I do dump at that point there are still 3 pins present, one for
>>>>>each
>>>>>PF, although they are all zombies - no parents as their parent pins are
>>>>>not
>>>>>registered (as the other patch [1/3] prevents dump of pin parent if the
>>>>>parent is not registered). Maybe we can remove the REGISTERED mark for
>>>>>all
>>>>>the muxed pins, if all their parents have been unregistered, so they
>>>>>won't
>>>>>be visible to the user at all. Will try to POC that.
>>>>>
>>>>>>>>
>>>>>>>>> directly connected pins with a dpll device. If unregistered parent
>>>>>>>>> determines if the muxed pin can be register with it or not, it
>>>>>>>>>forces
>>>>>>>>> serialized driver load order - first the driver instance which
>>>>>>>>> registers the direct pins needs to be loaded, then the other
>>>>>>>>> instances
>>>>>>>>> could register muxed type pins.
>>>>>>>>>
>>>>>>>>> Allow registration of a pin with a parent even if the parent was
>>>>>>>>>not
>>>>>>>>> yet registered, thus allow ability for unserialized driver instance
>>>>>>>>
>>>>>>>> Weird.
>>>>>>>>
>>>>>>>
>>>>>>> Yeah, this is issue only for MUX/parent pin part, couldn't find
>>>>>>>better
>>>>>>> way, but it doesn't seem to break things around..
>>>>>>>
>>>>>>
>>>>>>I just wonder how do you see the registration procedure? How can parent
>>>>>>pin exist if it's not registered? I believe you cannot get it through
>>>>>>DPLL API, then the only possible way is to create it within the same
>>>>>>driver code, which can be simply re-arranged. Am I wrong here?
>>>>>>
>>>>>
>>>>>By "parent exist" I mean the parent pin exist in the dpll subsystem
>>>>>(allocated on pins xa), but it doesn't mean it is available to the
>>>>>users,
>>>>>as it might not be registered with a dpll device.
>>>>>
>>>>>We have this 2 step init approach:
>>>>>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>>>>>2.1. dpll_pin_register(..) -> register with a dpll device
>>>>>2.2. dpll_pin_on_pin_register -> register with a parent pin
>>>>>
>>>>>Basically:
>>>>>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for
>>>>>its
>>>>>  recovery clock pin,
>>>>>- other PF's only do step 1 for the direct input pins (as they must get
>>>>>  reference to those in order to register recovery clock pin with them),
>>>>>  and steps: 1 & 2.2 for their recovery clock pin.
>>>>>
>>>>>
>>>>>Thank you!
>>>>>Arkadiusz
>>>>>
>>>>>>> Thank you!
>>>>>>> Arkadiusz
>>>>>>>
>>>>>>>>
>>>>>>>>> load order.
>>>>>>>>> Do not WARN_ON notification for unregistered pin, which can be
>>>>>>>>> invoked
>>>>>>>>> for described case, instead just return error.
>>>>>>>>>
>>>>>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>functions")
>>>>>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>> functions")
>>>>>>>>> Signed-off-by: Arkadiusz Kubalewski
>>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>>> ---
>>>>>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>> index
>>>>>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>>>>>> --- a/drivers/dpll/dpll_core.c
>>>>>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id,
>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>>
>>>>>>>>> struct dpll_device_registration {
>>>>>>>>> 	struct list_head list;
>>>>>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>> *parent,
>>>>>>>>> struct dpll_pin *pin,
>>>>>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>>>>>> 	    WARN_ON(!ops->direction_get))
>>>>>>>>> 		return -EINVAL;
>>>>>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>>>>>> -		return -EINVAL;
>>>>>>>>>
>>>>>>>>> 	mutex_lock(&dpll_lock);
>>>>>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops,
>>>>>>>>> priv); diff
>>>>>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>>>> index
>>>>>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event, struct
>>>>>>>>> dpll_pin *pin)
>>>>>>>>> 	int ret = -ENOMEM;
>>>>>>>>> 	void *hdr;
>>>>>>>>>
>>>>>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id,
>>>>>>>>> DPLL_REGISTERED)))
>>>>>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>>>>>> 		return -ENODEV;
>>>>>>>>>
>>>>>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>>>>>> --
>>>>>>>>> 2.38.1
>>>>>>>>>
>>>>>>
>>>>>

Re: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Jiri Pirko]

Fri, Nov 10, 2023 at 10:06:59AM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Friday, November 10, 2023 7:49 AM
>>
>>Fri, Nov 10, 2023 at 12:35:43AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Thursday, November 9, 2023 7:07 PM
>>>>
>>>>Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>>Sent: Thursday, November 9, 2023 11:51 AM
>>>>>>
>>>>>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>>>>>> Fix issues when performing unordered unbind/bind of a kernel modules
>>>>>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>>>>>> Currently only serialized bind/unbind of such use case works, fix
>>>>>>> the issues and allow for unserialized kernel module bind order.
>>>>>>>
>>>>>>> The issues are observed on the ice driver, i.e.,
>>>>>>>
>>>>>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>>>>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>>>>>
>>>>>>> results in:
>>>>>>>
>>>>>>> ice 0000:af:00.0: Removed PTP clock
>>>>>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>>>>>> PF: supervisor read access in kernel mode
>>>>>>> PF: error_code(0x0000) - not-present page
>>>>>>> PGD 0 P4D 0
>>>>>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>>>>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-
>>>>>>>rc5_next-
>>>>>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>>>>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>>>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>>>>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c 8b
>>>>>>>66
>>>>>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6 5b
>>>>>>>10
>>>>>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>>>>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>>>>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>>>>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>>>>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>>>>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>>>>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>>>>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>>>>>>knlGS:0000000000000000
>>>>>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>>>>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>>>>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>>>>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>>>>>> PKRU: 55555554
>>>>>>> Call Trace:
>>>>>>>   <TASK>
>>>>>>>   ? __die+0x20/0x70
>>>>>>>   ? page_fault_oops+0x76/0x170
>>>>>>>   ? exc_page_fault+0x65/0x150
>>>>>>>   ? asm_exc_page_fault+0x22/0x30
>>>>>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>>>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>>>>>   dpll_pin_event_send+0x7d/0x150
>>>>>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>>>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>>>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>>>>>   ice_remove+0xcd/0x200 [ice]
>>>>>>>   pci_device_remove+0x33/0xa0
>>>>>>>   device_release_driver_internal+0x193/0x200
>>>>>>>   unbind_store+0x9d/0xb0
>>>>>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>>>>>   vfs_write+0x2bb/0x3e0
>>>>>>>   ksys_write+0x5f/0xe0
>>>>>>>   do_syscall_64+0x59/0x90
>>>>>>>   ? filp_close+0x1b/0x30
>>>>>>>   ? do_dup2+0x7d/0xd0
>>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>>>>>> RIP: 0033:0x7fdc7d93eb97
>>>>>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f
>>>>>>>1e
>>>>>>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00
>>>>>>>f0
>>>>>>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>>>>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX:
>>>>>>>0000000000000001
>>>>>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>>>>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>>>>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>>>>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>>>>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>>>>>   </TASK>
>>>>>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1
>>>>>>>vfio
>>>>>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>>>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs
>>>>>>>libcrc32c
>>>>>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod
>>>>>>>target_core_mod
>>>>>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm
>>>>>>>intel_rapl_msr
>>>>>>>intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common
>>>>>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif x86_pkg_temp_thermal
>>>>>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>>>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr
>>>>>>>i2c_i801
>>>>>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>>>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4 mbcache
>>>>>>>jbd2
>>>>>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper ice
>>>>>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e libahci
>>>>>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded:
>>>>>>>iavf]
>>>>>>> CR2: 0000000000000010
>>>>>>>
>>>>>>> Arkadiusz Kubalewski (3):
>>>>>>>    dpll: fix pin dump crash after module unbind
>>>>>>>    dpll: fix pin dump crash for rebound module
>>>>>>>    dpll: fix register pin with unregistered parent pin
>>>>>>>
>>>>>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>>>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++------------
>>>>>>>--
>>>>>>>-
>>>>>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>>>>>
>>>>>>
>>>>>>
>>>>>>I still don't get how can we end up with unregistered pin. And
>>>>>>shouldn't
>>>>>>drivers do unregister of dpll/pin during release procedure? I thought
>>>>>>it
>>>>>>was kind of agreement we reached while developing the subsystem.
>>>>>>
>>>>>
>>>>>It's definitely not about ending up with unregistered pins.
>>>>>
>>>>>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in
>>>>>opposite
>>>>>order: PF3, PF2, PF1, PF0. And this is working without any issues.
>>>>
>>>>Please fix this in the driver.
>>>>
>>>
>>>Thanks for your feedback, but this is already wrong advice.
>>>
>>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>>wrong.
>>>As you might recall from our sync meetings, the dpll subsystem is to unify
>>>approaches and reduce the code in the drivers, where your advice is
>>>exactly
>>>opposite, suggested fix would require to implement extra synchronization
>>>of the
>>>dpll and pin registration state between driver instances, most probably
>>>with
>>>use of additional modules like aux-bus or something similar, which was
>>>from the
>>>very beginning something we tried to avoid.
>>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>>doesn't allow unbind the driver which have registered dpll and pins
>>>without
>>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>>driver.
>>
>>I replied in the other patch thread.
>>
>
>Yes, so did I.
>But what is the reason you have moved the discussion from the other thread
>into this one?

I didn't, not sure why you say so. I just wanted to make sure you
follow.

>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>>
>>>>>Above crash is caused because of unordered driver unload, where dpll
>>>>>subsystem
>>>>>tries to notify muxed pin was deleted, but at that time the parent is
>>>>>already
>>>>>gone, thus data points to memory which is no longer available, thus
>>>>>crash
>>>>>happens when trying to dump pin parents.
>>>>>
>>>>>This series fixes all issues I could find connected to the situation
>>>>>where
>>>>>muxed-pins are trying to access their parents, when parent registerer
>>>>>was
>>>>>removed
>>>>>in the meantime.
>>>>>
>>>>>Thank you!
>>>>>Arkadiusz

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 11:06 AM
>
>Fri, Nov 10, 2023 at 10:01:50AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Friday, November 10, 2023 7:46 AM
>>>
>>>Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Thursday, November 9, 2023 7:06 PM
>>>>>
>>>>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>>>>
>>>>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>>>wrote:
>>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>>>>
>>>>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>>>>>wrote:
>>>>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>>>>entirely
>>>>>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>>>>>that
>>>>>>>>>>pin), and kernel module is again bound, the pin properties would
>>>>>>>>>>not
>>>>>>>>>>be
>>>>>>>>>>updated (the properties are only assigned when memory for the pin
>>>>>>>>>>is
>>>>>>>>>>allocated), prop pointer still points to the kernel module memory
>>>>>>>>>>of
>>>>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>>>>
>>>>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>>>>crash.
>>>>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>>>>subsystem,
>>>>>>>>>>copy the content from the kernel module when pin is allocated,
>>>>>>>>>>instead
>>>>>>>>>>of
>>>>>>>>>>using memory of the kernel module.
>>>>>>>>>>
>>>>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>>functions")
>>>>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>>>functions")
>>>>>>>>>>Signed-off-by: Arkadiusz Kubalewski
>>>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>>>>---
>>>>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>>>>
>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx,
>>>>>>>>>>struct
>>>>>>>>>>module *module,
>>>>>>>>>> 		ret = -EINVAL;
>>>>>>>>>> 		goto err;
>>>>>>>>>> 	}
>>>>>>>>>>-	pin->prop = prop;
>>>>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>>>>
>>>>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>>>>
>>>>>>>>
>>>>>>>>Well, true. Need a fix.
>>>>>>>>Wondering if copying idea is better than just assigning prop pointer
>>>>>>>>on
>>>>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>>>>
>>>>>>>Not sure what do you mean. Examples please.
>>>>>>>
>>>>>>
>>>>>>Sure,
>>>>>>
>>>>>>Basically this change:
>>>>>>
>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>index ae884b92d68c..06b72d5877c3 100644
>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct
>>>>>>module
>>>>>>*module,
>>>>>>                    pos->pin_idx == pin_idx &&
>>>>>>                    pos->module == module) {
>>>>>>                        ret = pos;
>>>>>>+                       pos->prop = prop;
>>>>>>                        refcount_inc(&ret->refcount);
>>>>>>                        break;
>>>>>>                }
>>>>>>
>>>>>>would replace whole of this patch changes, although seems a bit hacky.
>>>>>
>>>>>Or event better, as I suggested in the other patch reply, resolve this
>>>>>internally in the driver registering things only when they are valid.
>>>>>Much better then to hack anything in dpll core.
>>>>>
>>>>
>>>>This approach seemed to me hacky, that is why started with coping the
>>>>data.
>>>>It is not about registering, rather about unregistering on driver
>>>>unbind, which brakes things, and currently cannot be recovered in
>>>>described case.
>>>
>>>Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
>>>related object. Very clean and simple.
>>>
>>
>>What you are suggesting is:
>>- special purpose bus in the driver,
>
>No, it is a simple notificator. Very common infra over the whole
>kernel code.
>

No difference if this is simple or not, it is special purpose.

>
>>- dpll-related,
>
>Is this the only thing that PF0 is special with? Perhaps you can
>utilize this for other features as well, since your fw design is like
>this.
>

None api user is allowed to the unordered driver bind when there are muxed
pins involved.
This requires a fix in the api not in the driver.

>
>>- not needed,
>>- prone for errors.
>>
>>The dpll subsystem is here to make driver life easier.
>
>No, the subsystem is never here to handle device specific issues. And
>your PF0 dependency is very clearly something device specific. Don't
>pollute the dpll subsystem with workaround to handle specific device
>needs. create/register the dplls objects from your driver only when it
>is valid to do so. Make sure the lifetime of such object stays in
>the scope of validity. Handle that in the driver. Very clear and simple.
>

In our case this is PF0 but this is broader issue. The muxed pins
infrastructure in now slightly broken and I am fixing it.
From the beginning it was designed to allow separated driver instances
to create a device and connect their pins with it.
Any driver which would use it would face this issue. What you are trying
to imply is that it is better to put traffic lights on each car instead
of putting them on crossroads.

>Thanks!
>

Thank you!
Arkadiusz

>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>>
>>>>>>Thank you!
>>>>>>Arkadiusz
>>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>>Thank you!
>>>>>>>>Arkadiusz
>>>>>>>>
>>>>>>>>>
>>>>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>>>*parent,
>>>>>>>>>>struct dpll_pin *pin,
>>>>>>>>>> 	unsigned long i, stop;
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>
>>>>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>>>>  * @module:		module of creator
>>>>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was
>>>>>>>>>>registered
>>>>>>>>>>with
>>>>>>>>>>  * @parent_refs:	hold references to parent pins pin was
>>>>>>>>>>registered
>>>>>>>>>>with
>>>>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>>>>>clock
>>>>>>>>>>from it
>>>>>>>>>>  * @refcount:		refcount
>>>>>>>>>>  **/
>>>>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>>>>> 	struct module *module;
>>>>>>>>>> 	struct xarray dpll_refs;
>>>>>>>>>> 	struct xarray parent_refs;
>>>>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>>>>> 	refcount_t refcount;
>>>>>>>>>> };
>>>>>>>>>>
>>>>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>>>>struct
>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>>>>>&freq,
>>>>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>>>>> 		return -EMSGSIZE;
>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>>>>> 		if (!nest)
>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>>>>sizeof(freq),
>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>> 		}
>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>>>>sizeof(freq),
>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>>>>>dpll_pin
>>>>>>>>>>*pin, u32 freq)
>>>>>>>>>> {
>>>>>>>>>> 	int fs;
>>>>>>>>>>
>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>>>>> 			return true;
>>>>>>>>>> 	return false;
>>>>>>>>>> }
>>>>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>>>>> {
>>>>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin
>>>>>>>>>>*pin,
>>>>>>>>>>u32
>>>>>>>>>>parent_idx,
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>> 	}
>>>>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>>>>struct
>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>> 	}
>>>>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll,
>>>>>>>>>>struct
>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>> 	}
>>>>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>>>>struct
>>>>>>>>>>dpll_device *dpll,
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>>>>allowed");
>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>> 	}
>>>>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>>>>struct
>>>>>>>>>>nlattr *phase_adj_attr,
>>>>>>>>>> 	int ret;
>>>>>>>>>>
>>>>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>>>>> 				    "phase adjust value not supported");
>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>>>>*mod_name_attr,
>>>>>>>>>> 	unsigned long i;
>>>>>>>>>>
>>>>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>>>>>-		prop = pin->prop;
>>>>>>>>>>+		prop = &pin->prop;
>>>>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>>>>--
>>>>>>>>>>2.38.1
>>>>>>>>>>
>>>>>>>>
>>>>

RE: [PATCH net 3/3] dpll: fix register pin with unregistered parent pin

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 11:07 AM
>
>Fri, Nov 10, 2023 at 09:50:34AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Friday, November 10, 2023 7:48 AM
>>>
>>>Fri, Nov 10, 2023 at 12:21:11AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Thursday, November 9, 2023 7:04 PM
>>>>>
>>>>>Thu, Nov 09, 2023 at 05:02:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>>>Sent: Thursday, November 9, 2023 11:56 AM
>>>>>>>To: Kubalewski, Arkadiusz <arkadiusz.kubalewski@intel.com>; Jiri
>>>>>>>Pirko
>>>>>>>
>>>>>>>On 09/11/2023 09:59, Kubalewski, Arkadiusz wrote:
>>>>>>>>> From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>> Sent: Wednesday, November 8, 2023 4:08 PM
>>>>>>>>>
>>>>>>>>> Wed, Nov 08, 2023 at 11:32:26AM CET,
>>>>>>>>>arkadiusz.kubalewski@intel.com
>>>>>>>>> wrote:
>>>>>>>>>> In case of multiple kernel module instances using the same dpll
>>>>>>>>>>device:
>>>>>>>>>> if only one registers dpll device, then only that one can
>>>>>>>>>>register
>>>>>>>>>
>>>>>>>>> They why you don't register in multiple instances? See mlx5 for a
>>>>>>>>> reference.
>>>>>>>>>
>>>>>>>>
>>>>>>>> Every registration requires ops, but for our case only PF0 is able
>>>>>>>> to
>>>>>>>> control dpll pins and device, thus only this can provide ops.
>>>>>>>> Basically without PF0, dpll is not able to be controlled, as well
>>>>>>>> as directly connected pins.
>>>>>>>>
>>>>>>>But why do you need other pins then, if FP0 doesn't exist?
>>>>>>>
>>>>>>
>>>>>>In general we don't need them at that point, but this is a corner
>>>>>>case,
>>>>>>where users for some reason decided to unbind PF 0, and I treat this
>>>>>>state
>>>>>>as temporary, where dpll/pins controllability is temporarily broken.
>>>>>
>>>>>So resolve this broken situation internally in the driver, registering
>>>>>things only in case PF0 is present. Some simple notification infra
>>>>>would
>>>>>do. Don't drag this into the subsystem internals.
>>>>>
>>>>
>>>>Thanks for your feedback, but this is already wrong advice.
>>>>
>>>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>>>wrong.
>>>>As you might recall from our sync meetings, the dpll subsystem is to
>>>>unify
>>>>approaches and reduce the code in the drivers, where your advice is
>>>>exactly
>>>
>>>No. Your driver knows when what objects are valid or not. Of a pin of
>>>PF1 is not valid because the "master" PF0 is gone, it is responsibility
>>>of your driver to resolve that. Don't bring this internal dependencies
>>>to the dpll core please, does not make any sense to do so. Thanks!
>>>
>>
>>No, a driver doesn't know it, those are separated instances, and you
>>already
>>suggested to implement special notification bus in the driver.
>>This is not needed and prone for another errors. The dpll subsystem is
>>here to
>>make driver life easier.
>
>See the other thread for my reply.
>

Ok, will do.

Thank you!
Arkadiusz

>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>opposite, suggested fix would require to implement extra synchronization
>>>>of the
>>>>dpll and pin registration state between driver instances, most probably
>>>>with
>>>>use of additional modules like aux-bus or something similar, which was
>>>>from the
>>>>very beginning something we tried to avoid.
>>>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>>>doesn't allow unbind the driver which have registered dpll and pins
>>>>without
>>>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>>>driver.
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>>
>>>>>>The dpll at that point is not registered, all the direct pins are also
>>>>>>not registered, thus not available to the users.
>>>>>>
>>>>>>When I do dump at that point there are still 3 pins present, one for
>>>>>>each
>>>>>>PF, although they are all zombies - no parents as their parent pins
>>>>>>are
>>>>>>not
>>>>>>registered (as the other patch [1/3] prevents dump of pin parent if
>>>>>>the
>>>>>>parent is not registered). Maybe we can remove the REGISTERED mark for
>>>>>>all
>>>>>>the muxed pins, if all their parents have been unregistered, so they
>>>>>>won't
>>>>>>be visible to the user at all. Will try to POC that.
>>>>>>
>>>>>>>>>
>>>>>>>>>> directly connected pins with a dpll device. If unregistered
>>>>>>>>>>parent
>>>>>>>>>> determines if the muxed pin can be register with it or not, it
>>>>>>>>>>forces
>>>>>>>>>> serialized driver load order - first the driver instance which
>>>>>>>>>> registers the direct pins needs to be loaded, then the other
>>>>>>>>>> instances
>>>>>>>>>> could register muxed type pins.
>>>>>>>>>>
>>>>>>>>>> Allow registration of a pin with a parent even if the parent was
>>>>>>>>>>not
>>>>>>>>>> yet registered, thus allow ability for unserialized driver
>>>>>>>>>>instance
>>>>>>>>>
>>>>>>>>> Weird.
>>>>>>>>>
>>>>>>>>
>>>>>>>> Yeah, this is issue only for MUX/parent pin part, couldn't find
>>>>>>>>better
>>>>>>>> way, but it doesn't seem to break things around..
>>>>>>>>
>>>>>>>
>>>>>>>I just wonder how do you see the registration procedure? How can
>>>>>>>parent
>>>>>>>pin exist if it's not registered? I believe you cannot get it through
>>>>>>>DPLL API, then the only possible way is to create it within the same
>>>>>>>driver code, which can be simply re-arranged. Am I wrong here?
>>>>>>>
>>>>>>
>>>>>>By "parent exist" I mean the parent pin exist in the dpll subsystem
>>>>>>(allocated on pins xa), but it doesn't mean it is available to the
>>>>>>users,
>>>>>>as it might not be registered with a dpll device.
>>>>>>
>>>>>>We have this 2 step init approach:
>>>>>>1. dpll_pin_get(..) -> allocate new pin or increase reference if exist
>>>>>>2.1. dpll_pin_register(..) -> register with a dpll device
>>>>>>2.2. dpll_pin_on_pin_register -> register with a parent pin
>>>>>>
>>>>>>Basically:
>>>>>>- PF 0 does 1 & 2.1 for all the direct inputs, and steps: 1 & 2.2 for
>>>>>>its
>>>>>>  recovery clock pin,
>>>>>>- other PF's only do step 1 for the direct input pins (as they must
>>>>>>get
>>>>>>  reference to those in order to register recovery clock pin with
>>>>>>them),
>>>>>>  and steps: 1 & 2.2 for their recovery clock pin.
>>>>>>
>>>>>>
>>>>>>Thank you!
>>>>>>Arkadiusz
>>>>>>
>>>>>>>> Thank you!
>>>>>>>> Arkadiusz
>>>>>>>>
>>>>>>>>>
>>>>>>>>>> load order.
>>>>>>>>>> Do not WARN_ON notification for unregistered pin, which can be
>>>>>>>>>> invoked
>>>>>>>>>> for described case, instead just return error.
>>>>>>>>>>
>>>>>>>>>> Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>>functions")
>>>>>>>>>> Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>>> functions")
>>>>>>>>>> Signed-off-by: Arkadiusz Kubalewski
>>>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>>>> ---
>>>>>>>>>> drivers/dpll/dpll_core.c    | 4 ----
>>>>>>>>>> drivers/dpll/dpll_netlink.c | 2 +-
>>>>>>>>>> 2 files changed, 1 insertion(+), 5 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>>> index
>>>>>>>>>> 4077b562ba3b..ae884b92d68c 100644
>>>>>>>>>> --- a/drivers/dpll/dpll_core.c
>>>>>>>>>> +++ b/drivers/dpll/dpll_core.c
>>>>>>>>>> @@ -28,8 +28,6 @@ static u32 dpll_xa_id;
>>>>>>>>>> 	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>>> #define ASSERT_DPLL_NOT_REGISTERED(d)	\
>>>>>>>>>> 	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id,
>>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>>> -#define ASSERT_PIN_REGISTERED(p)	\
>>>>>>>>>> -	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id,
>>>>>>>>>> DPLL_REGISTERED))
>>>>>>>>>>
>>>>>>>>>> struct dpll_device_registration {
>>>>>>>>>> 	struct list_head list;
>>>>>>>>>> @@ -641,8 +639,6 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>>> *parent,
>>>>>>>>>> struct dpll_pin *pin,
>>>>>>>>>> 	    WARN_ON(!ops->state_on_pin_get) ||
>>>>>>>>>> 	    WARN_ON(!ops->direction_get))
>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>> -	if (ASSERT_PIN_REGISTERED(parent))
>>>>>>>>>> -		return -EINVAL;
>>>>>>>>>>
>>>>>>>>>> 	mutex_lock(&dpll_lock);
>>>>>>>>>> 	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops,
>>>>>>>>>> priv); diff
>>>>>>>>>> --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
>>>>>>>>>> index
>>>>>>>>>> 963bbbbe6660..ff430f43304f 100644
>>>>>>>>>> --- a/drivers/dpll/dpll_netlink.c
>>>>>>>>>> +++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>>> @@ -558,7 +558,7 @@ dpll_pin_event_send(enum dpll_cmd event,
>>>>>>>>>>struct
>>>>>>>>>> dpll_pin *pin)
>>>>>>>>>> 	int ret = -ENOMEM;
>>>>>>>>>> 	void *hdr;
>>>>>>>>>>
>>>>>>>>>> -	if (WARN_ON(!xa_get_mark(&dpll_pin_xa, pin->id,
>>>>>>>>>> DPLL_REGISTERED)))
>>>>>>>>>> +	if (!xa_get_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED))
>>>>>>>>>> 		return -ENODEV;
>>>>>>>>>>
>>>>>>>>>> 	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
>>>>>>>>>> --
>>>>>>>>>> 2.38.1
>>>>>>>>>>
>>>>>>>
>>>>>>

RE: [PATCH net 0/3] dpll: fix unordered unbind/bind registerer issues

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 11:09 AM
>
>Fri, Nov 10, 2023 at 10:06:59AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Friday, November 10, 2023 7:49 AM
>>>
>>>Fri, Nov 10, 2023 at 12:35:43AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Thursday, November 9, 2023 7:07 PM
>>>>>
>>>>>Thu, Nov 09, 2023 at 06:20:14PM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>>From: Vadim Fedorenko <vadim.fedorenko@linux.dev>
>>>>>>>Sent: Thursday, November 9, 2023 11:51 AM
>>>>>>>
>>>>>>>On 08/11/2023 10:32, Arkadiusz Kubalewski wrote:
>>>>>>>> Fix issues when performing unordered unbind/bind of a kernel
>>>>>>>>modules
>>>>>>>> which are using a dpll device with DPLL_PIN_TYPE_MUX pins.
>>>>>>>> Currently only serialized bind/unbind of such use case works, fix
>>>>>>>> the issues and allow for unserialized kernel module bind order.
>>>>>>>>
>>>>>>>> The issues are observed on the ice driver, i.e.,
>>>>>>>>
>>>>>>>> $ echo 0000:af:00.0 > /sys/bus/pci/drivers/ice/unbind
>>>>>>>> $ echo 0000:af:00.1 > /sys/bus/pci/drivers/ice/unbind
>>>>>>>>
>>>>>>>> results in:
>>>>>>>>
>>>>>>>> ice 0000:af:00.0: Removed PTP clock
>>>>>>>> BUG: kernel NULL pointer dereference, address: 0000000000000010
>>>>>>>> PF: supervisor read access in kernel mode
>>>>>>>> PF: error_code(0x0000) - not-present page
>>>>>>>> PGD 0 P4D 0
>>>>>>>> Oops: 0000 [#1] PREEMPT SMP PTI
>>>>>>>> CPU: 7 PID: 71848 Comm: bash Kdump: loaded Not tainted 6.6.0-
>>>>>>>>rc5_next-
>>>>>>>>queue_19th-Oct-2023-01625-g039e5d15e451 #1
>>>>>>>> Hardware name: Intel Corporation S2600STB/S2600STB, BIOS
>>>>>>>>SE5C620.86B.02.01.0008.031920191559 03/19/2019
>>>>>>>> RIP: 0010:ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>>>> Code: 41 57 4d 89 cf 41 56 41 55 4d 89 c5 41 54 55 48 89 f5 53 4c
>>>>>>>>8b
>>>>>>>>66
>>>>>>>>08 48 89 cb 4d 8d b4 24 f0 49 00 00 4c 89 f7 e8 71 ec 1f c5 <0f> b6
>>>>>>>>5b
>>>>>>>>10
>>>>>>>>41 0f b6 84 24 30 4b 00 00 29 c3 41 0f b6 84 24 28 4b
>>>>>>>> RSP: 0018:ffffc902b179fb60 EFLAGS: 00010246
>>>>>>>> RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
>>>>>>>> RDX: ffff8882c1398000 RSI: ffff888c7435cc60 RDI: ffff888c7435cb90
>>>>>>>> RBP: ffff888c7435cc60 R08: ffffc902b179fbb0 R09: 0000000000000000
>>>>>>>> R10: ffff888ef1fc8050 R11: fffffffffff82700 R12: ffff888c743581a0
>>>>>>>> R13: ffffc902b179fbb0 R14: ffff888c7435cb90 R15: 0000000000000000
>>>>>>>> FS:  00007fdc7dae0740(0000) GS:ffff888c105c0000(0000)
>>>>>>>>knlGS:0000000000000000
>>>>>>>> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
>>>>>>>> CR2: 0000000000000010 CR3: 0000000132c24002 CR4: 00000000007706e0
>>>>>>>> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
>>>>>>>> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
>>>>>>>> PKRU: 55555554
>>>>>>>> Call Trace:
>>>>>>>>   <TASK>
>>>>>>>>   ? __die+0x20/0x70
>>>>>>>>   ? page_fault_oops+0x76/0x170
>>>>>>>>   ? exc_page_fault+0x65/0x150
>>>>>>>>   ? asm_exc_page_fault+0x22/0x30
>>>>>>>>   ? ice_dpll_rclk_state_on_pin_get+0x2f/0x90 [ice]
>>>>>>>>   ? __pfx_ice_dpll_rclk_state_on_pin_get+0x10/0x10 [ice]
>>>>>>>>   dpll_msg_add_pin_parents+0x142/0x1d0
>>>>>>>>   dpll_pin_event_send+0x7d/0x150
>>>>>>>>   dpll_pin_on_pin_unregister+0x3f/0x100
>>>>>>>>   ice_dpll_deinit_pins+0xa1/0x230 [ice]
>>>>>>>>   ice_dpll_deinit+0x29/0xe0 [ice]
>>>>>>>>   ice_remove+0xcd/0x200 [ice]
>>>>>>>>   pci_device_remove+0x33/0xa0
>>>>>>>>   device_release_driver_internal+0x193/0x200
>>>>>>>>   unbind_store+0x9d/0xb0
>>>>>>>>   kernfs_fop_write_iter+0x128/0x1c0
>>>>>>>>   vfs_write+0x2bb/0x3e0
>>>>>>>>   ksys_write+0x5f/0xe0
>>>>>>>>   do_syscall_64+0x59/0x90
>>>>>>>>   ? filp_close+0x1b/0x30
>>>>>>>>   ? do_dup2+0x7d/0xd0
>>>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>>>   ? syscall_exit_work+0x103/0x130
>>>>>>>>   ? syscall_exit_to_user_mode+0x22/0x40
>>>>>>>>   ? do_syscall_64+0x69/0x90
>>>>>>>>   entry_SYSCALL_64_after_hwframe+0x6e/0xd8
>>>>>>>> RIP: 0033:0x7fdc7d93eb97
>>>>>>>> Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3
>>>>>>>>0f
>>>>>>>>1e
>>>>>>>>fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d
>>>>>>>>00
>>>>>>>>f0
>>>>>>>>ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
>>>>>>>> RSP: 002b:00007fff2aa91028 EFLAGS: 00000246 ORIG_RAX:
>>>>>>>>0000000000000001
>>>>>>>> RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fdc7d93eb97
>>>>>>>> RDX: 000000000000000d RSI: 00005644814ec9b0 RDI: 0000000000000001
>>>>>>>> RBP: 00005644814ec9b0 R08: 0000000000000000 R09: 00007fdc7d9b14e0
>>>>>>>> R10: 00007fdc7d9b13e0 R11: 0000000000000246 R12: 000000000000000d
>>>>>>>> R13: 00007fdc7d9fb780 R14: 000000000000000d R15: 00007fdc7d9f69e0
>>>>>>>>   </TASK>
>>>>>>>> Modules linked in: uinput vfio_pci vfio_pci_core vfio_iommu_type1
>>>>>>>>vfio
>>>>>>>>irqbypass ixgbevf snd_seq_dummy snd_hrtimer snd_seq snd_timer
>>>>>>>>snd_seq_device snd soundcore overlay qrtr rfkill vfat fat xfs
>>>>>>>>libcrc32c
>>>>>>>>rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod
>>>>>>>>target_core_mod
>>>>>>>>ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm
>>>>>>>>intel_rapl_msr
>>>>>>>>intel_rapl_common intel_uncore_frequency
>>>>>>>>intel_uncore_frequency_common
>>>>>>>>isst_if_common skx_edac nfit libnvdimm ipmi_ssif
>>>>>>>>x86_pkg_temp_thermal
>>>>>>>>intel_powerclamp coretemp irdma rapl intel_cstate ib_uverbs iTCO_wdt
>>>>>>>>iTCO_vendor_support acpi_ipmi intel_uncore mei_me ipmi_si pcspkr
>>>>>>>>i2c_i801
>>>>>>>>ib_core mei ipmi_devintf intel_pch_thermal ioatdma i2c_smbus
>>>>>>>>ipmi_msghandler lpc_ich joydev acpi_power_meter acpi_pad ext4
>>>>>>>>mbcache
>>>>>>>>jbd2
>>>>>>>>sd_mod t10_pi sg ast i2c_algo_bit drm_shmem_helper drm_kms_helper
>>>>>>>>ice
>>>>>>>>crct10dif_pclmul ixgbe crc32_pclmul drm crc32c_intel ahci i40e
>>>>>>>>libahci
>>>>>>>>ghash_clmulni_intel libata mdio dca gnss wmi fuse [last unloaded:
>>>>>>>>iavf]
>>>>>>>> CR2: 0000000000000010
>>>>>>>>
>>>>>>>> Arkadiusz Kubalewski (3):
>>>>>>>>    dpll: fix pin dump crash after module unbind
>>>>>>>>    dpll: fix pin dump crash for rebound module
>>>>>>>>    dpll: fix register pin with unregistered parent pin
>>>>>>>>
>>>>>>>>   drivers/dpll/dpll_core.c    |  8 ++------
>>>>>>>>   drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>>   drivers/dpll/dpll_netlink.c | 37 ++++++++++++++++++++++----------
>>>>>>>>--
>>>>>>>>--
>>>>>>>>-
>>>>>>>>   3 files changed, 26 insertions(+), 23 deletions(-)
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>I still don't get how can we end up with unregistered pin. And
>>>>>>>shouldn't
>>>>>>>drivers do unregister of dpll/pin during release procedure? I thought
>>>>>>>it
>>>>>>>was kind of agreement we reached while developing the subsystem.
>>>>>>>
>>>>>>
>>>>>>It's definitely not about ending up with unregistered pins.
>>>>>>
>>>>>>Usually the driver is loaded for PF0, PF1, PF2, PF3 and unloaded in
>>>>>>opposite
>>>>>>order: PF3, PF2, PF1, PF0. And this is working without any issues.
>>>>>
>>>>>Please fix this in the driver.
>>>>>
>>>>
>>>>Thanks for your feedback, but this is already wrong advice.
>>>>
>>>>Our HW/FW is designed in different way than yours, it doesn't mean it is
>>>>wrong.
>>>>As you might recall from our sync meetings, the dpll subsystem is to
>>>>unify
>>>>approaches and reduce the code in the drivers, where your advice is
>>>>exactly
>>>>opposite, suggested fix would require to implement extra synchronization
>>>>of the
>>>>dpll and pin registration state between driver instances, most probably
>>>>with
>>>>use of additional modules like aux-bus or something similar, which was
>>>>from the
>>>>very beginning something we tried to avoid.
>>>>Only ice uses the infrastructure of muxed pins, and this is broken as it
>>>>doesn't allow unbind the driver which have registered dpll and pins
>>>>without
>>>>crashing the kernel, so a fix is required in dpll subsystem, not in the
>>>>driver.
>>>
>>>I replied in the other patch thread.
>>>
>>
>>Yes, so did I.
>>But what is the reason you have moved the discussion from the other thread
>>into this one?
>
>I didn't, not sure why you say so. I just wanted to make sure you
>follow.
>

Funny thing, you said you did not, and just after explained why.

Thank you!
Arkadiusz

>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>>
>>>>>>Above crash is caused because of unordered driver unload, where dpll
>>>>>>subsystem
>>>>>>tries to notify muxed pin was deleted, but at that time the parent is
>>>>>>already
>>>>>>gone, thus data points to memory which is no longer available, thus
>>>>>>crash
>>>>>>happens when trying to dump pin parents.
>>>>>>
>>>>>>This series fixes all issues I could find connected to the situation
>>>>>>where
>>>>>>muxed-pins are trying to access their parents, when parent registerer
>>>>>>was
>>>>>>removed
>>>>>>in the meantime.
>>>>>>
>>>>>>Thank you!
>>>>>>Arkadiusz

Re: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Jiri Pirko]

Fri, Nov 10, 2023 at 12:18:51PM CET, arkadiusz.kubalewski@intel.com wrote:
>>From: Jiri Pirko <jiri@resnulli.us>
>>Sent: Friday, November 10, 2023 11:06 AM
>>
>>Fri, Nov 10, 2023 at 10:01:50AM CET, arkadiusz.kubalewski@intel.com wrote:
>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>Sent: Friday, November 10, 2023 7:46 AM
>>>>
>>>>Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com
>>>>wrote:
>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>Sent: Thursday, November 9, 2023 7:06 PM
>>>>>>
>>>>>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com
>>>>>>wrote:
>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>>>>>
>>>>>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>>>>wrote:
>>>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>>>>>
>>>>>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET, arkadiusz.kubalewski@intel.com
>>>>>>>>>>wrote:
>>>>>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>>>>>entirely
>>>>>>>>>>>freed (other kernel module instance have had kept the reference to
>>>>>>>>>>>that
>>>>>>>>>>>pin), and kernel module is again bound, the pin properties would
>>>>>>>>>>>not
>>>>>>>>>>>be
>>>>>>>>>>>updated (the properties are only assigned when memory for the pin
>>>>>>>>>>>is
>>>>>>>>>>>allocated), prop pointer still points to the kernel module memory
>>>>>>>>>>>of
>>>>>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>>>>>
>>>>>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>>>>>crash.
>>>>>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>>>>>subsystem,
>>>>>>>>>>>copy the content from the kernel module when pin is allocated,
>>>>>>>>>>>instead
>>>>>>>>>>>of
>>>>>>>>>>>using memory of the kernel module.
>>>>>>>>>>>
>>>>>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>>>functions")
>>>>>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>>>>functions")
>>>>>>>>>>>Signed-off-by: Arkadiusz Kubalewski
>>>>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>>>>>---
>>>>>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>>>>>
>>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx,
>>>>>>>>>>>struct
>>>>>>>>>>>module *module,
>>>>>>>>>>> 		ret = -EINVAL;
>>>>>>>>>>> 		goto err;
>>>>>>>>>>> 	}
>>>>>>>>>>>-	pin->prop = prop;
>>>>>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>>>>>
>>>>>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>Well, true. Need a fix.
>>>>>>>>>Wondering if copying idea is better than just assigning prop pointer
>>>>>>>>>on
>>>>>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>>>>>
>>>>>>>>Not sure what do you mean. Examples please.
>>>>>>>>
>>>>>>>
>>>>>>>Sure,
>>>>>>>
>>>>>>>Basically this change:
>>>>>>>
>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>index ae884b92d68c..06b72d5877c3 100644
>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct
>>>>>>>module
>>>>>>>*module,
>>>>>>>                    pos->pin_idx == pin_idx &&
>>>>>>>                    pos->module == module) {
>>>>>>>                        ret = pos;
>>>>>>>+                       pos->prop = prop;
>>>>>>>                        refcount_inc(&ret->refcount);
>>>>>>>                        break;
>>>>>>>                }
>>>>>>>
>>>>>>>would replace whole of this patch changes, although seems a bit hacky.
>>>>>>
>>>>>>Or event better, as I suggested in the other patch reply, resolve this
>>>>>>internally in the driver registering things only when they are valid.
>>>>>>Much better then to hack anything in dpll core.
>>>>>>
>>>>>
>>>>>This approach seemed to me hacky, that is why started with coping the
>>>>>data.
>>>>>It is not about registering, rather about unregistering on driver
>>>>>unbind, which brakes things, and currently cannot be recovered in
>>>>>described case.
>>>>
>>>>Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
>>>>related object. Very clean and simple.
>>>>
>>>
>>>What you are suggesting is:
>>>- special purpose bus in the driver,
>>
>>No, it is a simple notificator. Very common infra over the whole
>>kernel code.
>>
>
>No difference if this is simple or not, it is special purpose.
>
>>
>>>- dpll-related,
>>
>>Is this the only thing that PF0 is special with? Perhaps you can
>>utilize this for other features as well, since your fw design is like
>>this.
>>
>
>None api user is allowed to the unordered driver bind when there are muxed
>pins involved.
>This requires a fix in the api not in the driver.
>
>>
>>>- not needed,
>>>- prone for errors.
>>>
>>>The dpll subsystem is here to make driver life easier.
>>
>>No, the subsystem is never here to handle device specific issues. And
>>your PF0 dependency is very clearly something device specific. Don't
>>pollute the dpll subsystem with workaround to handle specific device
>>needs. create/register the dplls objects from your driver only when it
>>is valid to do so. Make sure the lifetime of such object stays in
>>the scope of validity. Handle that in the driver. Very clear and simple.
>>
>
>In our case this is PF0 but this is broader issue. The muxed pins
>infrastructure in now slightly broken and I am fixing it.
>From the beginning it was designed to allow separated driver instances
>to create a device and connect their pins with it.

That is true. That's exacly what we have implemented in mlx5. Each
instance registers dpll device and the pin related to the instance. No
problem.

The fact that you do register only in PF0 is a limitation in your
driver. Fix it there.


>Any driver which would use it would face this issue. What you are trying
>to imply is that it is better to put traffic lights on each car instead
>of putting them on crossroads.
>
>>Thanks!
>>
>
>Thank you!
>Arkadiusz
>
>>
>>>
>>>Thank you!
>>>Arkadiusz
>>>
>>>>
>>>>>
>>>>>Thank you!
>>>>>Arkadiusz
>>>>>
>>>>>>
>>>>>>>
>>>>>>>Thank you!
>>>>>>>Arkadiusz
>>>>>>>
>>>>>>>>
>>>>>>>>>
>>>>>>>>>Thank you!
>>>>>>>>>Arkadiusz
>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>>>>*parent,
>>>>>>>>>>>struct dpll_pin *pin,
>>>>>>>>>>> 	unsigned long i, stop;
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>>
>>>>>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>>>>>  * @module:		module of creator
>>>>>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was
>>>>>>>>>>>registered
>>>>>>>>>>>with
>>>>>>>>>>>  * @parent_refs:	hold references to parent pins pin was
>>>>>>>>>>>registered
>>>>>>>>>>>with
>>>>>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>>>>>  * @rclk_dev_name:	holds name of device when pin can recover
>>>>>>>>>>>clock
>>>>>>>>>>>from it
>>>>>>>>>>>  * @refcount:		refcount
>>>>>>>>>>>  **/
>>>>>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>>>>>> 	struct module *module;
>>>>>>>>>>> 	struct xarray dpll_refs;
>>>>>>>>>>> 	struct xarray parent_refs;
>>>>>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>>>>>> 	refcount_t refcount;
>>>>>>>>>>> };
>>>>>>>>>>>
>>>>>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>>>>>struct
>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY, sizeof(freq),
>>>>>>>>>>>&freq,
>>>>>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>>>>>> 		return -EMSGSIZE;
>>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>>>>>> 		if (!nest)
>>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>>>>>sizeof(freq),
>>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>>> 		}
>>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>>>>>sizeof(freq),
>>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>>>@@ -304,9 +304,9 @@ static bool dpll_pin_is_freq_supported(struct
>>>>>>>>>>>dpll_pin
>>>>>>>>>>>*pin, u32 freq)
>>>>>>>>>>> {
>>>>>>>>>>> 	int fs;
>>>>>>>>>>>
>>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>>>>>> 			return true;
>>>>>>>>>>> 	return false;
>>>>>>>>>>> }
>>>>>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>>>>>> {
>>>>>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin
>>>>>>>>>>>*pin,
>>>>>>>>>>>u32
>>>>>>>>>>>parent_idx,
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>> 	}
>>>>>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>>>>>struct
>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not allowed");
>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>> 	}
>>>>>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll,
>>>>>>>>>>>struct
>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not allowed");
>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>> 	}
>>>>>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>>>>>struct
>>>>>>>>>>>dpll_device *dpll,
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>>>>>allowed");
>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>> 	}
>>>>>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>>>>>struct
>>>>>>>>>>>nlattr *phase_adj_attr,
>>>>>>>>>>> 	int ret;
>>>>>>>>>>>
>>>>>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>>>>>> 				    "phase adjust value not supported");
>>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>>>>>*mod_name_attr,
>>>>>>>>>>> 	unsigned long i;
>>>>>>>>>>>
>>>>>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED) {
>>>>>>>>>>>-		prop = pin->prop;
>>>>>>>>>>>+		prop = &pin->prop;
>>>>>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id : true;
>>>>>>>>>>> 		mod_match = mod_name_attr && module_name(pin->module) ?
>>>>>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>>>>>--
>>>>>>>>>>>2.38.1
>>>>>>>>>>>
>>>>>>>>>
>>>>>

RE: [PATCH net 2/3] dpll: fix pin dump crash for rebound module

[Kubalewski, Arkadiusz]

>From: Jiri Pirko <jiri@resnulli.us>
>Sent: Friday, November 10, 2023 12:45 PM
>
>Fri, Nov 10, 2023 at 12:18:51PM CET, arkadiusz.kubalewski@intel.com wrote:
>>>From: Jiri Pirko <jiri@resnulli.us>
>>>Sent: Friday, November 10, 2023 11:06 AM
>>>
>>>Fri, Nov 10, 2023 at 10:01:50AM CET, arkadiusz.kubalewski@intel.com
>>>wrote:
>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>Sent: Friday, November 10, 2023 7:46 AM
>>>>>
>>>>>Fri, Nov 10, 2023 at 12:32:21AM CET, arkadiusz.kubalewski@intel.com
>>>>>wrote:
>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>Sent: Thursday, November 9, 2023 7:06 PM
>>>>>>>
>>>>>>>Thu, Nov 09, 2023 at 05:30:20PM CET, arkadiusz.kubalewski@intel.com
>>>>>>>wrote:
>>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>>Sent: Thursday, November 9, 2023 2:19 PM
>>>>>>>>>
>>>>>>>>>Thu, Nov 09, 2023 at 01:20:48PM CET, arkadiusz.kubalewski@intel.com
>>>>>>>>>wrote:
>>>>>>>>>>>From: Jiri Pirko <jiri@resnulli.us>
>>>>>>>>>>>Sent: Wednesday, November 8, 2023 3:30 PM
>>>>>>>>>>>
>>>>>>>>>>>Wed, Nov 08, 2023 at 11:32:25AM CET,
>>>>>>>>>>>arkadiusz.kubalewski@intel.com
>>>>>>>>>>>wrote:
>>>>>>>>>>>>When a kernel module is unbound but the pin resources were not
>>>>>>>>>>>>entirely
>>>>>>>>>>>>freed (other kernel module instance have had kept the reference
>>>>>>>>>>>>to
>>>>>>>>>>>>that
>>>>>>>>>>>>pin), and kernel module is again bound, the pin properties would
>>>>>>>>>>>>not
>>>>>>>>>>>>be
>>>>>>>>>>>>updated (the properties are only assigned when memory for the
>>>>>>>>>>>>pin
>>>>>>>>>>>>is
>>>>>>>>>>>>allocated), prop pointer still points to the kernel module
>>>>>>>>>>>>memory
>>>>>>>>>>>>of
>>>>>>>>>>>>the kernel module which was deallocated on the unbind.
>>>>>>>>>>>>
>>>>>>>>>>>>If the pin dump is invoked in this state, the result is a kernel
>>>>>>>>>>>>crash.
>>>>>>>>>>>>Prevent the crash by storing persistent pin properties in dpll
>>>>>>>>>>>>subsystem,
>>>>>>>>>>>>copy the content from the kernel module when pin is allocated,
>>>>>>>>>>>>instead
>>>>>>>>>>>>of
>>>>>>>>>>>>using memory of the kernel module.
>>>>>>>>>>>>
>>>>>>>>>>>>Fixes: 9431063ad323 ("dpll: core: Add DPLL framework base
>>>>>>>>>>>>functions")
>>>>>>>>>>>>Fixes: 9d71b54b65b1 ("dpll: netlink: Add DPLL framework base
>>>>>>>>>>>>functions")
>>>>>>>>>>>>Signed-off-by: Arkadiusz Kubalewski
>>>>>>>>>>>><arkadiusz.kubalewski@intel.com>
>>>>>>>>>>>>---
>>>>>>>>>>>> drivers/dpll/dpll_core.c    |  4 ++--
>>>>>>>>>>>> drivers/dpll/dpll_core.h    |  4 ++--
>>>>>>>>>>>> drivers/dpll/dpll_netlink.c | 28 ++++++++++++++--------------
>>>>>>>>>>>> 3 files changed, 18 insertions(+), 18 deletions(-)
>>>>>>>>>>>>
>>>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>>>>>index 3568149b9562..4077b562ba3b 100644
>>>>>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>>>>>@@ -442,7 +442,7 @@ dpll_pin_alloc(u64 clock_id, u32 pin_idx,
>>>>>>>>>>>>struct
>>>>>>>>>>>>module *module,
>>>>>>>>>>>> 		ret = -EINVAL;
>>>>>>>>>>>> 		goto err;
>>>>>>>>>>>> 	}
>>>>>>>>>>>>-	pin->prop = prop;
>>>>>>>>>>>>+	memcpy(&pin->prop, prop, sizeof(pin->prop));
>>>>>>>>>>>
>>>>>>>>>>>Odd, you don't care about the pointer within this structure?
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>Well, true. Need a fix.
>>>>>>>>>>Wondering if copying idea is better than just assigning prop
>>>>>>>>>>pointer
>>>>>>>>>>on
>>>>>>>>>>each call to dpll_pin_get(..) function (when pin already exists)?
>>>>>>>>>
>>>>>>>>>Not sure what do you mean. Examples please.
>>>>>>>>>
>>>>>>>>
>>>>>>>>Sure,
>>>>>>>>
>>>>>>>>Basically this change:
>>>>>>>>
>>>>>>>>diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
>>>>>>>>index ae884b92d68c..06b72d5877c3 100644
>>>>>>>>--- a/drivers/dpll/dpll_core.c
>>>>>>>>+++ b/drivers/dpll/dpll_core.c
>>>>>>>>@@ -483,6 +483,7 @@ dpll_pin_get(u64 clock_id, u32 pin_idx, struct
>>>>>>>>module
>>>>>>>>*module,
>>>>>>>>                    pos->pin_idx == pin_idx &&
>>>>>>>>                    pos->module == module) {
>>>>>>>>                        ret = pos;
>>>>>>>>+                       pos->prop = prop;
>>>>>>>>                        refcount_inc(&ret->refcount);
>>>>>>>>                        break;
>>>>>>>>                }
>>>>>>>>
>>>>>>>>would replace whole of this patch changes, although seems a bit
>>>>>>>hacky.
>>>>>>>
>>>>>>>Or event better, as I suggested in the other patch reply, resolve
>>>>>>>this
>>>>>>>internally in the driver registering things only when they are valid.
>>>>>>>Much better then to hack anything in dpll core.
>>>>>>>
>>>>>>
>>>>>>This approach seemed to me hacky, that is why started with coping the
>>>>>>data.
>>>>>>It is not about registering, rather about unregistering on driver
>>>>>>unbind, which brakes things, and currently cannot be recovered in
>>>>>>described case.
>>>>>
>>>>>Sure it can. PF0 unbind-> internal notification-> PF1 unregisters all
>>>>>related object. Very clean and simple.
>>>>>
>>>>
>>>>What you are suggesting is:
>>>>- special purpose bus in the driver,
>>>
>>>No, it is a simple notificator. Very common infra over the whole
>>>kernel code.
>>>
>>
>>No difference if this is simple or not, it is special purpose.
>>
>>>
>>>>- dpll-related,
>>>
>>>Is this the only thing that PF0 is special with? Perhaps you can
>>>utilize this for other features as well, since your fw design is like
>>>this.
>>>
>>
>>None api user is allowed to the unordered driver bind when there are muxed
>>pins involved.
>>This requires a fix in the api not in the driver.
>>
>>>
>>>>- not needed,
>>>>- prone for errors.
>>>>
>>>>The dpll subsystem is here to make driver life easier.
>>>
>>>No, the subsystem is never here to handle device specific issues. And
>>>your PF0 dependency is very clearly something device specific. Don't
>>>pollute the dpll subsystem with workaround to handle specific device
>>>needs. create/register the dplls objects from your driver only when it
>>>is valid to do so. Make sure the lifetime of such object stays in
>>>the scope of validity. Handle that in the driver. Very clear and simple.
>>>
>>
>>In our case this is PF0 but this is broader issue. The muxed pins
>>infrastructure in now slightly broken and I am fixing it.
>>From the beginning it was designed to allow separated driver instances
>>to create a device and connect their pins with it.
>
>That is true. That's exacly what we have implemented in mlx5. Each
>instance registers dpll device and the pin related to the instance. No
>problem.
>

I read this: "mlx5 is done correctly, you need to follow", funny again.

>The fact that you do register only in PF0 is a limitation in your
>driver. Fix it there.
>

A limitation when comparing to your design, makes it not the limitation at all.
Different design is just different.

According to documentation there is no such thing like a correct order of
registering/unregistering pins.
So a driver can register a parent pin, register a pin with that parent, and
remove the parent first. It shall not crash. Right now it is broken. The
dpll needs to handle such cases. I am fixing it. Simple.

Thank you!
Arkadiusz

>
>>Any driver which would use it would face this issue. What you are trying
>>to imply is that it is better to put traffic lights on each car instead
>>of putting them on crossroads.
>>
>>>Thanks!
>>>
>>
>>Thank you!
>>Arkadiusz
>>
>>>
>>>>
>>>>Thank you!
>>>>Arkadiusz
>>>>
>>>>>
>>>>>>
>>>>>>Thank you!
>>>>>>Arkadiusz
>>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>>Thank you!
>>>>>>>>Arkadiusz
>>>>>>>>
>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>Thank you!
>>>>>>>>>>Arkadiusz
>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>> 	refcount_set(&pin->refcount, 1);
>>>>>>>>>>>> 	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
>>>>>>>>>>>> 	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
>>>>>>>>>>>>@@ -634,7 +634,7 @@ int dpll_pin_on_pin_register(struct dpll_pin
>>>>>>>>>>>>*parent,
>>>>>>>>>>>>struct dpll_pin *pin,
>>>>>>>>>>>> 	unsigned long i, stop;
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>>-	if (WARN_ON(parent->prop->type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>>>>+	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
>>>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>>>
>>>>>>>>>>>> 	if (WARN_ON(!ops) ||
>>>>>>>>>>>>diff --git a/drivers/dpll/dpll_core.h b/drivers/dpll/dpll_core.h
>>>>>>>>>>>>index 5585873c5c1b..717f715015c7 100644
>>>>>>>>>>>>--- a/drivers/dpll/dpll_core.h
>>>>>>>>>>>>+++ b/drivers/dpll/dpll_core.h
>>>>>>>>>>>>@@ -44,7 +44,7 @@ struct dpll_device {
>>>>>>>>>>>>  * @module:		module of creator
>>>>>>>>>>>>  * @dpll_refs:		hold referencees to dplls pin was
>>>>>>>>>>>>registered
>>>>>>>>>>>>with
>>>>>>>>>>>>  * @parent_refs:	hold references to parent pins pin was
>>>>>>>>>>>>registered
>>>>>>>>>>>>with
>>>>>>>>>>>>- * @prop:		pointer to pin properties given by registerer
>>>>>>>>>>>>+ * @prop:		pin properties copied from the registerer
>>>>>>>>>>>>  * @rclk_dev_name:	holds name of device when pin can
>>>>>>>>>>>>recover
>>>>>>>>>>>>clock
>>>>>>>>>>>>from it
>>>>>>>>>>>>  * @refcount:		refcount
>>>>>>>>>>>>  **/
>>>>>>>>>>>>@@ -55,7 +55,7 @@ struct dpll_pin {
>>>>>>>>>>>> 	struct module *module;
>>>>>>>>>>>> 	struct xarray dpll_refs;
>>>>>>>>>>>> 	struct xarray parent_refs;
>>>>>>>>>>>>-	const struct dpll_pin_properties *prop;
>>>>>>>>>>>>+	struct dpll_pin_properties prop;
>>>>>>>>>>>> 	refcount_t refcount;
>>>>>>>>>>>> };
>>>>>>>>>>>>
>>>>>>>>>>>>diff --git a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>>b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>>index 93fc6c4b8a78..963bbbbe6660 100644
>>>>>>>>>>>>--- a/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>>+++ b/drivers/dpll/dpll_netlink.c
>>>>>>>>>>>>@@ -278,17 +278,17 @@ dpll_msg_add_pin_freq(struct sk_buff *msg,
>>>>>>>>>>>>struct
>>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>>> 	if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY,
>>>>>>>>>>>>sizeof(freq),
>>>>>>>>>>>>&freq,
>>>>>>>>>>>> 			  DPLL_A_PIN_PAD))
>>>>>>>>>>>> 		return -EMSGSIZE;
>>>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++) {
>>>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++) {
>>>>>>>>>>>> 		nest = nla_nest_start(msg,
>>>>>>>>>>>>DPLL_A_PIN_FREQUENCY_SUPPORTED);
>>>>>>>>>>>> 		if (!nest)
>>>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].min;
>>>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].min;
>>>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MIN,
>>>>>>>>>>>>sizeof(freq),
>>>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>>>> 			return -EMSGSIZE;
>>>>>>>>>>>> 		}
>>>>>>>>>>>>-		freq = pin->prop->freq_supported[fs].max;
>>>>>>>>>>>>+		freq = pin->prop.freq_supported[fs].max;
>>>>>>>>>>>> 		if (nla_put_64bit(msg, DPLL_A_PIN_FREQUENCY_MAX,
>>>>>>>>>>>>sizeof(freq),
>>>>>>>>>>>> 				  &freq, DPLL_A_PIN_PAD)) {
>>>>>>>>>>>> 			nla_nest_cancel(msg, nest);
>>>>>>>>>>>>@@ -304,9 +304,9 @@ static bool
>>>>>>>>>>>>dpll_pin_is_freq_supported(struct
>>>>>>>>>>>>dpll_pin
>>>>>>>>>>>>*pin, u32 freq)
>>>>>>>>>>>> {
>>>>>>>>>>>> 	int fs;
>>>>>>>>>>>>
>>>>>>>>>>>>-	for (fs = 0; fs < pin->prop->freq_supported_num; fs++)
>>>>>>>>>>>>-		if (freq >= pin->prop->freq_supported[fs].min &&
>>>>>>>>>>>>-		    freq <= pin->prop->freq_supported[fs].max)
>>>>>>>>>>>>+	for (fs = 0; fs < pin->prop.freq_supported_num; fs++)
>>>>>>>>>>>>+		if (freq >= pin->prop.freq_supported[fs].min &&
>>>>>>>>>>>>+		    freq <= pin->prop.freq_supported[fs].max)
>>>>>>>>>>>> 			return true;
>>>>>>>>>>>> 	return false;
>>>>>>>>>>>> }
>>>>>>>>>>>>@@ -403,7 +403,7 @@ static int
>>>>>>>>>>>> dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
>>>>>>>>>>>> 		     struct netlink_ext_ack *extack)
>>>>>>>>>>>> {
>>>>>>>>>>>>-	const struct dpll_pin_properties *prop = pin->prop;
>>>>>>>>>>>>+	const struct dpll_pin_properties *prop = &pin->prop;
>>>>>>>>>>>> 	struct dpll_pin_ref *ref;
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>>@@ -696,7 +696,7 @@ dpll_pin_on_pin_state_set(struct dpll_pin
>>>>>>>>>>>>*pin,
>>>>>>>>>>>>u32
>>>>>>>>>>>>parent_idx,
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not
>>>>>>>>>>>>allowed");
>>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>>> 	}
>>>>>>>>>>>>@@ -732,7 +732,7 @@ dpll_pin_state_set(struct dpll_device *dpll,
>>>>>>>>>>>>struct
>>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE &
>>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "state changing is not
>>>>>>>>>>>>allowed");
>>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>>> 	}
>>>>>>>>>>>>@@ -759,7 +759,7 @@ dpll_pin_prio_set(struct dpll_device *dpll,
>>>>>>>>>>>>struct
>>>>>>>>>>>>dpll_pin *pin,
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE &
>>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "prio changing is not
>>>>>>>>>>>>allowed");
>>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>>> 	}
>>>>>>>>>>>>@@ -787,7 +787,7 @@ dpll_pin_direction_set(struct dpll_pin *pin,
>>>>>>>>>>>>struct
>>>>>>>>>>>>dpll_device *dpll,
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>> 	if (!(DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE &
>>>>>>>>>>>>-	      pin->prop->capabilities)) {
>>>>>>>>>>>>+	      pin->prop.capabilities)) {
>>>>>>>>>>>> 		NL_SET_ERR_MSG(extack, "direction changing is not
>>>>>>>>>>>>allowed");
>>>>>>>>>>>> 		return -EOPNOTSUPP;
>>>>>>>>>>>> 	}
>>>>>>>>>>>>@@ -817,8 +817,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin,
>>>>>>>>>>>>struct
>>>>>>>>>>>>nlattr *phase_adj_attr,
>>>>>>>>>>>> 	int ret;
>>>>>>>>>>>>
>>>>>>>>>>>> 	phase_adj = nla_get_s32(phase_adj_attr);
>>>>>>>>>>>>-	if (phase_adj > pin->prop->phase_range.max ||
>>>>>>>>>>>>-	    phase_adj < pin->prop->phase_range.min) {
>>>>>>>>>>>>+	if (phase_adj > pin->prop.phase_range.max ||
>>>>>>>>>>>>+	    phase_adj < pin->prop.phase_range.min) {
>>>>>>>>>>>> 		NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr,
>>>>>>>>>>>> 				    "phase adjust value not
>>>>>>>>>>>>supported");
>>>>>>>>>>>> 		return -EINVAL;
>>>>>>>>>>>>@@ -999,7 +999,7 @@ dpll_pin_find(u64 clock_id, struct nlattr
>>>>>>>>>>>>*mod_name_attr,
>>>>>>>>>>>> 	unsigned long i;
>>>>>>>>>>>>
>>>>>>>>>>>> 	xa_for_each_marked(&dpll_pin_xa, i, pin, DPLL_REGISTERED)
>>>>>>>>>>>>{
>>>>>>>>>>>>-		prop = pin->prop;
>>>>>>>>>>>>+		prop = &pin->prop;
>>>>>>>>>>>> 		cid_match = clock_id ? pin->clock_id == clock_id :
>>>>>>>>>>>>true;
>>>>>>>>>>>> 		mod_match = mod_name_attr && module_name(pin-
>>>>>>>>>>>>module) ?
>>>>>>>>>>>> 			!nla_strcmp(mod_name_attr,
>>>>>>>>>>>>--
>>>>>>>>>>>>2.38.1
>>>>>>>>>>>>
>>>>>>>>>>
>>>>>>