From: Chaitra P B <chaitra.basappa@broadcom.com>
To: linux-scsi@vger.kernel.org, stable@vger.kernel.org
Cc: suganath-prabu.subramani@broadcom.com,
Sathya.Prakash@broadcom.com, sreekanth.reddy@broadcom.com,
Chaitra P B <chaitra.basappa@broadcom.com>
Subject: [PATCH 05/15] mpt3sas: Optimize I/O memory consumption in driver.
Date: Fri, 30 Mar 2018 15:07:14 +0530 [thread overview]
Message-ID: <1522402644-3016-6-git-send-email-chaitra.basappa@broadcom.com> (raw)
In-Reply-To: <1522402644-3016-1-git-send-email-chaitra.basappa@broadcom.com>
For every IO, memory of PAGE size is allocated for handling NVMe native
PRPS. And in addition to that for every IO (chains need per IO * chain
buffer size, e.g. 38 * 128byte) amount of memory is allocated for chain
buffers.
However, at any point of time; the IO request can be for NVMe target device
(where PRP's page is used for framing PRP's) or can be for SCSI target
device (where chain buffers are used for framing chain SGE's). This patch
modifies the driver to reuse same pre-allocated PRP page buffers as a chain
buffer for IO's targeted for SCSI target devices. No need to allocate
separate buffers for chain SGE's buffers.
Suppose if the number of chain buffers need for IO doesn't fit in the PRP
Page size then driver maintain's separate buffers for those extra chain
buffers that exceeds the PRP page size. For example consider PRP page size
as 4K and chain buffer size as 128 bytes, then number of chain buffers that
can fit in PRP page is 4096/128 => 32. if the number of chain buffer need
per IO exceeds 32; for example consider number of chains need per IO is 36
then for remaining 4 chain buffer's driver allocates them individual.
Signed-off-by: Chaitra P B <chaitra.basappa@broadcom.com>
Signed-off-by: Suganath Prabu S <suganath-prabu.subramani@broadcom.com>
---
drivers/scsi/mpt3sas/mpt3sas_base.c | 80 +++++++++++++++++++++++--------------
1 file changed, 51 insertions(+), 29 deletions(-)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 1e8e399..701e1e7 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -4188,7 +4188,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
kfree(ioc->internal_lookup);
if (ioc->chain_lookup) {
for (i = 0; i < ioc->scsiio_depth; i++) {
- for (j = 0; j < ioc->chains_needed_per_io; j++) {
+ for (j = ioc->chains_per_prp_buffer;
+ j < ioc->chains_needed_per_io; j++) {
ct = &ioc->chain_lookup[i].chains_per_smid[j];
if (ct && ct->chain_buffer)
dma_pool_free(ioc->chain_dma_pool,
@@ -4506,7 +4507,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
if (!ioc->chain_lookup) {
pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages "
- "failed\n", ioc->name);
+ "failed\n", ioc->name);
goto out;
}
@@ -4520,33 +4521,6 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
}
}
- ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
- ioc->chain_segment_sz, 16, 0);
- if (!ioc->chain_dma_pool) {
- pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
- ioc->name);
- goto out;
- }
- for (i = 0; i < ioc->scsiio_depth; i++) {
- for (j = 0; j < ioc->chains_needed_per_io; j++) {
- ct = &ioc->chain_lookup[i].chains_per_smid[j];
- ct->chain_buffer = dma_pool_alloc(
- ioc->chain_dma_pool , GFP_KERNEL,
- &ct->chain_buffer_dma);
- if (!ct->chain_buffer) {
- pr_err(MPT3SAS_FMT "chain_lookup: "
- " pci_pool_alloc failed\n", ioc->name);
- goto out;
- }
- }
- total_sz += ioc->chain_segment_sz;
- }
-
- dinitprintk(ioc, pr_info(MPT3SAS_FMT
- "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
- ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
- ((ioc->chain_depth * ioc->chain_segment_sz))/1024));
-
/* initialize hi-priority queue smid's */
ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
sizeof(struct request_tracker), GFP_KERNEL);
@@ -4587,6 +4561,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
* be required for NVMe PRP's, only each set of NVMe blocks will be
* contiguous, so a new set is allocated for each possible I/O.
*/
+ ioc->chains_per_prp_buffer = 0;
if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
nvme_blocks_needed =
(ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
@@ -4609,6 +4584,11 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
ioc->name);
goto out;
}
+
+ ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
+ ioc->chains_per_prp_buffer = min(ioc->chains_per_prp_buffer,
+ ioc->chains_needed_per_io);
+
for (i = 0; i < ioc->scsiio_depth; i++) {
ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
ioc->pcie_sgl_dma_pool, GFP_KERNEL,
@@ -4619,13 +4599,55 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
ioc->name);
goto out;
}
+ for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
+ ct = &ioc->chain_lookup[i].chains_per_smid[j];
+ ct->chain_buffer =
+ ioc->pcie_sg_lookup[i].pcie_sgl +
+ (j * ioc->chain_segment_sz);
+ ct->chain_buffer_dma =
+ ioc->pcie_sg_lookup[i].pcie_sgl_dma +
+ (j * ioc->chain_segment_sz);
+ }
}
dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
"element_size(%d), pool_size(%d kB)\n", ioc->name,
ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
+ dinitprintk(ioc, pr_info(MPT3SAS_FMT "Number of chains can "
+ "fit in a PRP page(%d)\n", ioc->name,
+ ioc->chains_per_prp_buffer));
total_sz += sz * ioc->scsiio_depth;
}
+
+ ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
+ ioc->chain_segment_sz, 16, 0);
+ if (!ioc->chain_dma_pool) {
+ pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
+ ioc->name);
+ goto out;
+ }
+ for (i = 0; i < ioc->scsiio_depth; i++) {
+ for (j = ioc->chains_per_prp_buffer;
+ j < ioc->chains_needed_per_io; j++) {
+ ct = &ioc->chain_lookup[i].chains_per_smid[j];
+ ct->chain_buffer = dma_pool_alloc(
+ ioc->chain_dma_pool, GFP_KERNEL,
+ &ct->chain_buffer_dma);
+ if (!ct->chain_buffer) {
+ pr_err(MPT3SAS_FMT "chain_lookup: "
+ " pci_pool_alloc failed\n", ioc->name);
+ _base_release_memory_pools(ioc);
+ goto out;
+ }
+ }
+ total_sz += ioc->chain_segment_sz;
+ }
+
+ dinitprintk(ioc, pr_info(MPT3SAS_FMT
+ "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
+ ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
+ ((ioc->chain_depth * ioc->chain_segment_sz))/1024));
+
/* sense buffers, 4 byte align */
sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
--
1.8.3.1
next prev parent reply other threads:[~2018-03-30 9:39 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-30 9:37 [PATCH 00/15] mpt3sas: Enhancements and Defect fixes Chaitra P B
2018-03-30 9:37 ` [PATCH 01/15] mpt3sas: Fixed warnings Chaitra P B
2018-03-30 9:37 ` [PATCH 02/15] mpt3sas: Pre-allocate RDPQ Array at driver boot time Chaitra P B
2018-03-30 9:37 ` [PATCH 03/15] mpt3sas: Add sanity checks for scsi tracker before accessing it Chaitra P B
2018-03-30 11:59 ` Christoph Hellwig
2018-04-02 6:23 ` Sreekanth Reddy
2018-04-02 15:25 ` Bart Van Assche
2018-04-03 4:41 ` Sreekanth Reddy
2018-04-03 15:56 ` Bart Van Assche
2018-04-04 12:28 ` Sreekanth Reddy
2018-03-30 15:54 ` Bart Van Assche
2018-03-30 9:37 ` [PATCH 04/15] mpt3sas: Lockless access for chain buffers Chaitra P B
2018-03-30 9:37 ` Chaitra P B [this message]
2018-03-30 9:37 ` [PATCH 06/15] mpt3sas: Enhanced handling of Sense Buffer Chaitra P B
2018-03-30 9:37 ` [PATCH 07/15] mpt3sas: Added support for SAS Device Discovery Error Event Chaitra P B
2018-03-30 9:37 ` [PATCH 08/15] mpt3sas: Increase event log buffer to support 24 port HBA's Chaitra P B
2018-03-30 9:37 ` [PATCH 09/15] mpt3sas: Allow processing of events during driver unload Chaitra P B
2018-03-30 9:37 ` [PATCH 10/15] mpt3sas: Cache enclosure pages during enclosure add Chaitra P B
2018-03-30 9:37 ` [PATCH 11/15] mpt3sas: Report Firmware Package Version from HBA Driver Chaitra P B
2018-03-31 8:44 ` kbuild test robot
2018-03-30 9:37 ` [PATCH 12/15] mpt3sas: Update MPI Headers Chaitra P B
2018-03-30 9:37 ` [PATCH 13/15] mpt3sas: For NVME device, issue a protocol level reset instead of hot reset and use TM timeout value exposed in PCIe Device Page 2 Chaitra P B
2018-03-30 9:37 ` [PATCH 14/15] mpt3sas: fix possible memory leak Chaitra P B
2018-03-30 9:37 ` [PATCH 15/15] mpt3sas: Update driver version "25.100.00.00" Chaitra P B
2018-03-30 9:58 ` [PATCH 00/15] mpt3sas: Enhancements and Defect fixes Greg KH
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1522402644-3016-6-git-send-email-chaitra.basappa@broadcom.com \
--to=chaitra.basappa@broadcom.com \
--cc=Sathya.Prakash@broadcom.com \
--cc=linux-scsi@vger.kernel.org \
--cc=sreekanth.reddy@broadcom.com \
--cc=stable@vger.kernel.org \
--cc=suganath-prabu.subramani@broadcom.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox