Re: [smartpqi updates PATCH V2 02/11] smartpqi: add controller handshake during kdump

john.p.donnelly
2021-09-30T18:23:01Z

On 9/28/21 6:54 PM, Don Brace wrote:
> From: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
>
> Correct kdump hangs when controller is locked up.
>
> There are occasions when a controller reboot
> (controller soft reset) is issued when a controller
> firmware crash dump is in progress.
>
> This leads to incomplete controller firmware crash dump.
> - When the controller crash dump is in progress,
> and a kdump is initiated, the driver issues
> inbound doorbell reset to bring back the
> controller in SIS mode.
> - If the controller is in locked up state,
> the inbound doorbell reset does not work causing
> controller initialization failures. This results
> in the driver hanging waiting for SIS mode.
>
> To avoid an incomplete controller crash dump, add in
> a controller crash dump handshake.
> - Controller will indicate start and end of the controller
> crash dump by setting some register bits.
> - Driver will look these bits when a kdump is initiated.
> If a controller crash dump is in progress, the driver will
> wait for the controller crash dump to complete
> before issuing the controller soft reset then complete
> driver initialization.
>
> Reviewed-by: Scott Benesh <scott.benesh@microchip.com>
> Reviewed-by: Scott Teel <scott.teel@microchip.com>
> Reviewed-by: Mike McGowen <mike.mcgowen@microchip.com>
> Signed-off-by: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
> Signed-off-by: Don Brace <don.brace@microchip.com>

Acked-by: John Donnelly <john.p.donnelly@oracle.com>

> ---
> drivers/scsi/smartpqi/smartpqi_init.c | 41 +++++++++++++++++++--
> drivers/scsi/smartpqi/smartpqi_sis.c | 51 +++++++++++++++++++++++++++
> drivers/scsi/smartpqi/smartpqi_sis.h | 1 +
> 3 files changed, 91 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
> index 97027574eb1f..5655d240f7a7 100644
> --- a/drivers/scsi/smartpqi/smartpqi_init.c
> +++ b/drivers/scsi/smartpqi/smartpqi_init.c
> @@ -234,15 +234,46 @@ static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
> return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
> }
>
> +#define PQI_DRIVER_SCRATCH_PQI_MODE 0x1
> +#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED 0x2
> +
> static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
> {
> - return sis_read_driver_scratch(ctrl_info);
> + return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
> }
>
> static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
> enum pqi_ctrl_mode mode)
> {
> - sis_write_driver_scratch(ctrl_info, mode);
> + u32 driver_scratch;
> +
> + driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> + if (mode == PQI_MODE)
> + driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
> + else
> + driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
> +
> + sis_write_driver_scratch(ctrl_info, driver_scratch);
> +}
> +
> +static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
> +{
> + return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
> +}
> +
> +static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
> +{
> + u32 driver_scratch;
> +
> + driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> + if (is_supported)
> + driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> + else
> + driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> +
> + sis_write_driver_scratch(ctrl_info, driver_scratch);
> }
>
> static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
> @@ -7292,6 +7323,7 @@ static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
> ctrl_info->unique_wwid_in_report_phys_lun_supported =
> firmware_feature->enabled;
> break;
> + pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
> }
>
> pqi_firmware_feature_status(ctrl_info, firmware_feature);
> @@ -7618,6 +7650,11 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
> u32 product_id;
>
> if (reset_devices) {
> + if (pqi_is_fw_triage_supported(ctrl_info)) {
> + rc = sis_wait_for_fw_triage_completion(ctrl_info);
> + if (rc)
> + return rc;
> + }
> sis_soft_reset(ctrl_info);
> msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
> } else {
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
> index d63c46a8e38b..8acd3a80f582 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.c
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.c
> @@ -51,12 +51,20 @@
> #define SIS_BASE_STRUCT_REVISION 9
> #define SIS_BASE_STRUCT_ALIGNMENT 16
>
> +#define SIS_CTRL_KERNEL_FW_TRIAGE 0x3
> #define SIS_CTRL_KERNEL_UP 0x80
> #define SIS_CTRL_KERNEL_PANIC 0x100
> #define SIS_CTRL_READY_TIMEOUT_SECS 180
> #define SIS_CTRL_READY_RESUME_TIMEOUT_SECS 90
> #define SIS_CTRL_READY_POLL_INTERVAL_MSECS 10
>
> +enum sis_fw_triage_status {
> + FW_TRIAGE_NOT_STARTED = 0,
> + FW_TRIAGE_STARTED,
> + FW_TRIAGE_COND_INVALID,
> + FW_TRIAGE_COMPLETED
> +};
> +
> #pragma pack(1)
>
> /* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
> @@ -419,12 +427,55 @@ u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
> return readl(&ctrl_info->registers->sis_driver_scratch);
> }
>
> +static inline enum sis_fw_triage_status
> + sis_read_firmware_triage_status(struct pqi_ctrl_info *ctrl_info)
> +{
> + return ((enum sis_fw_triage_status)(readl(&ctrl_info->registers->sis_firmware_status) &
> + SIS_CTRL_KERNEL_FW_TRIAGE));
> +}
> +
> void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
> {
> writel(SIS_SOFT_RESET,
> &ctrl_info->registers->sis_host_to_ctrl_doorbell);
> }
>
> +#define SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS 300
> +#define SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS 1
> +
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info)
> +{
> + int rc;
> + enum sis_fw_triage_status status;
> + unsigned long timeout;
> +
> + timeout = (SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
> + while (1) {
> + status = sis_read_firmware_triage_status(ctrl_info);
> + if (status == FW_TRIAGE_COND_INVALID) {
> + dev_err(&ctrl_info->pci_dev->dev,
> + "firmware triage condition invalid\n");
> + rc = -EINVAL;
> + break;
> + } else if (status == FW_TRIAGE_NOT_STARTED ||
> + status == FW_TRIAGE_COMPLETED) {
> + rc = 0;
> + break;
> + }
> +
> + if (time_after(jiffies, timeout)) {
> + dev_err(&ctrl_info->pci_dev->dev,
> + "timed out waiting for firmware triage status\n");
> + rc = -ETIMEDOUT;
> + break;
> + }
> +
> + ssleep(SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS);
> + }
> +
> + return rc;
> +}
> +
> static void __attribute__((unused)) verify_structures(void)
> {
> BUILD_BUG_ON(offsetof(struct sis_base_struct,
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
> index d29c1352a826..c1db93054c86 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.h
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.h
> @@ -28,5 +28,6 @@ void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
> u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
> void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
> u32 sis_get_product_id(struct pqi_ctrl_info *ctrl_info);
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info);
>
> #endif /* _SMARTPQI_SIS_H */
>