From: James Smart <jsmart2021@gmail.com>
To: linux-scsi@vger.kernel.org
Cc: James Smart <jsmart2021@gmail.com>,
Dick Kennedy <dick.kennedy@broadcom.com>,
James Smart <james.smart@broadcom.com>
Subject: [PATCH v2 08/17] lpfc: Fix counters so outstandng NVME IO count is accurate
Date: Thu, 1 Jun 2017 21:07:02 -0700 [thread overview]
Message-ID: <20170602040711.21046-9-jsmart2021@gmail.com> (raw)
In-Reply-To: <20170602040711.21046-1-jsmart2021@gmail.com>
NVME FC counters don't reflect actual results
Since counters are not atomic, or protected by a lock,
the values often get screwed up.
Make them atomic, like NVMET.
Fix up sysfs and debugfs display accordingly
Added Outstanding IOs to stats display
Signed-off-by: Dick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: James Smart <james.smart@broadcom.com>
---
drivers/scsi/lpfc/lpfc.h | 20 ++++++++++----------
drivers/scsi/lpfc/lpfc_attr.c | 32 +++++++++++++++++++++-----------
drivers/scsi/lpfc/lpfc_debugfs.c | 30 +++++++++++++++++++++---------
drivers/scsi/lpfc/lpfc_init.c | 10 ++++++++++
drivers/scsi/lpfc/lpfc_nvme.c | 19 +++++++++++++------
drivers/scsi/lpfc/lpfc_scsi.c | 19 ++++++++++++++-----
6 files changed, 89 insertions(+), 41 deletions(-)
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index f2c0ba6ced78..a9d73728a68c 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -913,16 +913,16 @@ struct lpfc_hba {
/*
* stat counters
*/
- uint64_t fc4ScsiInputRequests;
- uint64_t fc4ScsiOutputRequests;
- uint64_t fc4ScsiControlRequests;
- uint64_t fc4ScsiIoCmpls;
- uint64_t fc4NvmeInputRequests;
- uint64_t fc4NvmeOutputRequests;
- uint64_t fc4NvmeControlRequests;
- uint64_t fc4NvmeIoCmpls;
- uint64_t fc4NvmeLsRequests;
- uint64_t fc4NvmeLsCmpls;
+ atomic_t fc4ScsiInputRequests;
+ atomic_t fc4ScsiOutputRequests;
+ atomic_t fc4ScsiControlRequests;
+ atomic_t fc4ScsiIoCmpls;
+ atomic_t fc4NvmeInputRequests;
+ atomic_t fc4NvmeOutputRequests;
+ atomic_t fc4NvmeControlRequests;
+ atomic_t fc4NvmeIoCmpls;
+ atomic_t fc4NvmeLsRequests;
+ atomic_t fc4NvmeLsCmpls;
uint64_t bg_guard_err_cnt;
uint64_t bg_apptag_err_cnt;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 200a614bb540..eb33473cbc62 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -150,6 +150,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
struct nvme_fc_local_port *localport;
struct lpfc_nodelist *ndlp;
struct nvme_fc_remote_port *nrport;
+ uint64_t data1, data2, data3, tot;
char *statep;
int len = 0;
@@ -244,11 +245,18 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
atomic_read(&tgtp->xmt_abort_rsp),
atomic_read(&tgtp->xmt_abort_rsp_error));
+ spin_lock(&phba->sli4_hba.nvmet_io_lock);
+ tot = phba->sli4_hba.nvmet_xri_cnt -
+ phba->sli4_hba.nvmet_ctx_cnt;
+ spin_unlock(&phba->sli4_hba.nvmet_io_lock);
+
len += snprintf(buf + len, PAGE_SIZE - len,
- "IO_CTX: %08x outstanding %08x total %x",
+ "IO_CTX: %08x WAIT: cur %08x tot %08x\n"
+ "CTX Outstanding %08llx\n",
phba->sli4_hba.nvmet_ctx_cnt,
phba->sli4_hba.nvmet_io_wait_cnt,
- phba->sli4_hba.nvmet_io_wait_total);
+ phba->sli4_hba.nvmet_io_wait_total,
+ tot);
len += snprintf(buf+len, PAGE_SIZE-len, "\n");
return len;
@@ -337,19 +345,21 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
len += snprintf(buf + len, PAGE_SIZE - len, "\nNVME Statistics\n");
len += snprintf(buf+len, PAGE_SIZE-len,
- "LS: Xmt %016llx Cmpl %016llx\n",
- phba->fc4NvmeLsRequests,
- phba->fc4NvmeLsCmpls);
-
+ "LS: Xmt %016x Cmpl %016x\n",
+ atomic_read(&phba->fc4NvmeLsRequests),
+ atomic_read(&phba->fc4NvmeLsCmpls));
+
+ tot = atomic_read(&phba->fc4NvmeIoCmpls);
+ data1 = atomic_read(&phba->fc4NvmeInputRequests);
+ data2 = atomic_read(&phba->fc4NvmeOutputRequests);
+ data3 = atomic_read(&phba->fc4NvmeControlRequests);
len += snprintf(buf+len, PAGE_SIZE-len,
"FCP: Rd %016llx Wr %016llx IO %016llx\n",
- phba->fc4NvmeInputRequests,
- phba->fc4NvmeOutputRequests,
- phba->fc4NvmeControlRequests);
+ data1, data2, data3);
len += snprintf(buf+len, PAGE_SIZE-len,
- " Cmpl %016llx\n", phba->fc4NvmeIoCmpls);
-
+ " Cmpl %016llx Outstanding %016llx\n",
+ tot, (data1 + data2 + data3) - tot);
return len;
}
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index dd8598bc50f3..6089690fb345 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -750,6 +750,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
struct lpfc_hba *phba = vport->phba;
struct lpfc_nvmet_tgtport *tgtp;
struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
+ uint64_t tot, data1, data2, data3;
int len = 0;
int cnt;
@@ -847,11 +848,18 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
}
+ spin_lock(&phba->sli4_hba.nvmet_io_lock);
+ tot = phba->sli4_hba.nvmet_xri_cnt -
+ phba->sli4_hba.nvmet_ctx_cnt;
+ spin_unlock(&phba->sli4_hba.nvmet_io_lock);
+
len += snprintf(buf + len, size - len,
- "IO_CTX: %08x outstanding %08x total %08x\n",
+ "IO_CTX: %08x WAIT: cur %08x tot %08x\n"
+ "CTX Outstanding %08llx\n",
phba->sli4_hba.nvmet_ctx_cnt,
phba->sli4_hba.nvmet_io_wait_cnt,
- phba->sli4_hba.nvmet_io_wait_total);
+ phba->sli4_hba.nvmet_io_wait_total,
+ tot);
} else {
if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
return len;
@@ -860,18 +868,22 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
"\nNVME Lport Statistics\n");
len += snprintf(buf + len, size - len,
- "LS: Xmt %016llx Cmpl %016llx\n",
- phba->fc4NvmeLsRequests,
- phba->fc4NvmeLsCmpls);
+ "LS: Xmt %016x Cmpl %016x\n",
+ atomic_read(&phba->fc4NvmeLsRequests),
+ atomic_read(&phba->fc4NvmeLsCmpls));
+
+ tot = atomic_read(&phba->fc4NvmeIoCmpls);
+ data1 = atomic_read(&phba->fc4NvmeInputRequests);
+ data2 = atomic_read(&phba->fc4NvmeOutputRequests);
+ data3 = atomic_read(&phba->fc4NvmeControlRequests);
len += snprintf(buf + len, size - len,
"FCP: Rd %016llx Wr %016llx IO %016llx\n",
- phba->fc4NvmeInputRequests,
- phba->fc4NvmeOutputRequests,
- phba->fc4NvmeControlRequests);
+ data1, data2, data3);
len += snprintf(buf + len, size - len,
- " Cmpl %016llx\n", phba->fc4NvmeIoCmpls);
+ " Cmpl %016llx Outstanding %016llx\n",
+ tot, (data1 + data2 + data3) - tot);
}
return len;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 9add9473cae5..3064f0768033 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -6731,6 +6731,16 @@ lpfc_create_shost(struct lpfc_hba *phba)
phba->fc_arbtov = FF_DEF_ARBTOV;
atomic_set(&phba->sdev_cnt, 0);
+ atomic_set(&phba->fc4ScsiInputRequests, 0);
+ atomic_set(&phba->fc4ScsiOutputRequests, 0);
+ atomic_set(&phba->fc4ScsiControlRequests, 0);
+ atomic_set(&phba->fc4ScsiIoCmpls, 0);
+ atomic_set(&phba->fc4NvmeInputRequests, 0);
+ atomic_set(&phba->fc4NvmeOutputRequests, 0);
+ atomic_set(&phba->fc4NvmeControlRequests, 0);
+ atomic_set(&phba->fc4NvmeIoCmpls, 0);
+ atomic_set(&phba->fc4NvmeLsRequests, 0);
+ atomic_set(&phba->fc4NvmeLsCmpls, 0);
vport = lpfc_create_port(phba, phba->brd_no, &phba->pcidev->dev);
if (!vport)
return -ENODEV;
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index ede42411e57b..8206aa5493e3 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -211,7 +211,7 @@ lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
struct lpfc_dmabuf *buf_ptr;
struct lpfc_nodelist *ndlp;
- vport->phba->fc4NvmeLsCmpls++;
+ atomic_inc(&vport->phba->fc4NvmeLsCmpls);
pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
@@ -478,7 +478,7 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
&pnvme_lsreq->rspdma);
- vport->phba->fc4NvmeLsRequests++;
+ atomic_inc(&vport->phba->fc4NvmeLsRequests);
/* Hardcode the wait to 30 seconds. Connections are failing otherwise.
* This code allows it all to work.
@@ -773,7 +773,7 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
wcqe);
return;
}
- phba->fc4NvmeIoCmpls++;
+ atomic_inc(&phba->fc4NvmeIoCmpls);
nCmd = lpfc_ncmd->nvmeCmd;
rport = lpfc_ncmd->nrport;
@@ -998,7 +998,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
NVME_WRITE_CMD);
- phba->fc4NvmeOutputRequests++;
+ atomic_inc(&phba->fc4NvmeOutputRequests);
} else {
/* Word 7 */
bf_set(wqe_cmnd, &wqe->generic.wqe_com,
@@ -1019,7 +1019,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
NVME_READ_CMD);
- phba->fc4NvmeInputRequests++;
+ atomic_inc(&phba->fc4NvmeInputRequests);
}
} else {
/* Word 4 */
@@ -1040,7 +1040,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
/* Word 11 */
bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD);
- phba->fc4NvmeControlRequests++;
+ atomic_inc(&phba->fc4NvmeControlRequests);
}
/*
* Finish initializing those WQE fields that are independent
@@ -1361,6 +1361,13 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
return 0;
out_free_nvme_buf:
+ if (lpfc_ncmd->nvmeCmd->sg_cnt) {
+ if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
+ atomic_dec(&phba->fc4NvmeOutputRequests);
+ else
+ atomic_dec(&phba->fc4NvmeInputRequests);
+ } else
+ atomic_dec(&phba->fc4NvmeControlRequests);
lpfc_release_nvme_buf(phba, lpfc_ncmd);
out_fail:
return ret;
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 54fd0c81ceaf..cfe1d01eb73f 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -3931,7 +3931,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
struct Scsi_Host *shost;
uint32_t logit = LOG_FCP;
- phba->fc4ScsiIoCmpls++;
+ atomic_inc(&phba->fc4ScsiIoCmpls);
/* Sanity check on return of outstanding command */
cmd = lpfc_cmd->pCmd;
@@ -4250,19 +4250,19 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
vport->cfg_first_burst_size;
}
fcp_cmnd->fcpCntl3 = WRITE_DATA;
- phba->fc4ScsiOutputRequests++;
+ atomic_inc(&phba->fc4ScsiOutputRequests);
} else {
iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR;
iocb_cmd->ulpPU = PARM_READ_CHECK;
fcp_cmnd->fcpCntl3 = READ_DATA;
- phba->fc4ScsiInputRequests++;
+ atomic_inc(&phba->fc4ScsiInputRequests);
}
} else {
iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR;
iocb_cmd->un.fcpi.fcpi_parm = 0;
iocb_cmd->ulpPU = 0;
fcp_cmnd->fcpCntl3 = 0;
- phba->fc4ScsiControlRequests++;
+ atomic_inc(&phba->fc4ScsiControlRequests);
}
if (phba->sli_rev == 3 &&
!(phba->sli3_options & LPFC_SLI3_BG_ENABLED))
@@ -4640,7 +4640,16 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
(uint32_t)
(cmnd->request->timeout / 1000));
-
+ switch (lpfc_cmd->fcp_cmnd->fcpCntl3) {
+ case WRITE_DATA:
+ atomic_dec(&phba->fc4ScsiOutputRequests);
+ break;
+ case READ_DATA:
+ atomic_dec(&phba->fc4ScsiInputRequests);
+ break;
+ default:
+ atomic_dec(&phba->fc4ScsiControlRequests);
+ }
goto out_host_busy_free_buf;
}
if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
--
2.11.0
next prev parent reply other threads:[~2017-06-02 4:07 UTC|newest]
Thread overview: 22+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-06-02 4:06 [PATCH v2 00/17] lpfc updates for 11.4.0.0 James Smart
2017-06-02 4:06 ` [PATCH v2 01/17] lpfc: Add nvme initiator devloss support James Smart
2017-06-02 4:06 ` [PATCH v2 02/17] lpfc: Fix transition nvme-i rport handling to nport only James Smart
2017-06-02 4:06 ` [PATCH v2 03/17] lpfc: Fix nvme port role handling in sysfs and debugfs handlers James Smart
2017-06-02 4:06 ` [PATCH v2 04/17] lpfc: Add changes to assist in NVMET debugging James Smart
2017-06-02 4:06 ` [PATCH v2 05/17] lpfc: Fix Lun Priority level shown as NA James Smart
2017-06-02 4:07 ` [PATCH v2 06/17] lpfc: Fix nvmet node ref count handling James Smart
2017-06-02 4:07 ` [PATCH v2 07/17] lpfc: Fix Port going offline after multiple resets James Smart
2017-06-02 4:07 ` James Smart [this message]
2017-06-02 4:07 ` [PATCH v2 09/17] lpfc: Fix return value of board_mode store routine in case of online failure James Smart
2017-06-02 4:07 ` [PATCH v2 10/17] lpfc: Fix crash on powering off BFS VM with passthrough device James Smart
2017-06-02 4:07 ` [PATCH v2 11/17] lpfc: Fix System panic after loading the driver James Smart
2017-06-02 4:07 ` [PATCH v2 12/17] lpfc: Null pointer dereference when log_verbose is set to 0xffffffff James Smart
2017-06-02 4:07 ` [PATCH v2 13/17] lpfc: Fix PRLI retry handling when target rejects it James Smart
2017-06-02 4:07 ` [PATCH v2 14/17] lpfc: Fix vports not logging into target James Smart
2017-06-02 4:07 ` [PATCH v2 15/17] lpfc: Fix defects reported by Coverity Scan James Smart
2017-06-02 4:07 ` [PATCH v2 16/17] lpfc: Add auto EQ delay logic James Smart
2017-06-02 4:07 ` [PATCH v2 17/17] lpfc: update to revision to 11.4.0.0 James Smart
2017-06-06 1:13 ` [PATCH v2 00/17] lpfc updates for 11.4.0.0 Martin K. Petersen
2017-06-07 4:20 ` James Smart
2017-06-13 3:04 ` Martin K. Petersen
2017-06-15 16:59 ` James Smart
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170602040711.21046-9-jsmart2021@gmail.com \
--to=jsmart2021@gmail.com \
--cc=dick.kennedy@broadcom.com \
--cc=james.smart@broadcom.com \
--cc=linux-scsi@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox