Hi all, Today's linux-next merge of the block tree got a conflict in: drivers/nvme/host/pci.c between commit: 95d2fec40c483 ("dma-mapping: rename dma_opt_mapping_size()") from the dma-mapping tree and commit: 2e8e30b3cf9f1 ("Revert "nvme-pci: clamp max_hw_sectors based on DMA optimized limitation"") from the block tree. I fixed it up (see below) and can carry the fix as necessary. This is now fixed as far as linux-next is concerned, but any non trivial conflicts should be mentioned to your upstream maintainer when your tree is submitted for merging. You may also want to consider cooperating with the maintainer of the conflicting tree to minimise any particularly complex conflicts. diff --combined drivers/nvme/host/pci.c index dbcfc7ddb78aa,ea552ac5501f6..0000000000000 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@@ -30,6 -30,7 +30,7 @@@ #include "trace.h" #include "nvme.h" + #include "pci-dmabuf.h" #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) @@@ -318,6 -319,7 +319,7 @@@ struct nvme_dev bool hmb; struct sg_table *hmb_sgt; mempool_t *dmavec_mempool; + struct nvme_dma_buf_list dmabufs; /* shadow doorbell buffer support: */ __le32 *dbbuf_dbs; @@@ -865,6 -867,76 +867,76 @@@ static void nvme_free_descriptors(struc } } + static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req, + struct nvme_queue *nvmeq) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_dmabuf_map *map = nvme_req_to_map(req); + unsigned bvec_done = req->bio->bi_iter.bi_offset; + unsigned map_idx = bvec_done / NVME_CTRL_PAGE_SIZE; + unsigned offset = bvec_done & (NVME_CTRL_PAGE_SIZE - 1); + int length = blk_rq_payload_bytes(req) - (NVME_CTRL_PAGE_SIZE - offset); + dma_addr_t *dma_list = map->dma_list; + u64 prp1_dma = dma_list[map_idx++] + offset; + u64 dma_addr, prp2_dma; + dma_addr_t prp_dma; + __le64 *prp_list; + unsigned i; + + if (length <= 0) { + prp2_dma = 0; + goto done; + } + + if (length <= NVME_CTRL_PAGE_SIZE) { + prp2_dma = dma_list[map_idx]; + goto done; + } + + if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <= + NVME_SMALL_POOL_SIZE / sizeof(__le64)) + iod->flags |= IOD_SMALL_DESCRIPTOR; + + prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, + &prp_dma); + if (!prp_list) + return BLK_STS_RESOURCE; + + iod->descriptors[iod->nr_descriptors++] = prp_list; + prp2_dma = prp_dma; + i = 0; + for (;;) { + if (i == NVME_CTRL_PAGE_SIZE >> 3) { + __le64 *old_prp_list = prp_list; + + prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large, + GFP_ATOMIC, &prp_dma); + if (!prp_list) + goto free_prps; + iod->descriptors[iod->nr_descriptors++] = prp_list; + prp_list[0] = old_prp_list[i - 1]; + old_prp_list[i - 1] = cpu_to_le64(prp_dma); + i = 1; + } + + dma_addr = dma_list[map_idx++]; + prp_list[i++] = cpu_to_le64(dma_addr); + + length -= NVME_CTRL_PAGE_SIZE; + if (length <= 0) + break; + } + done: + iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); + iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); + return BLK_STS_OK; + free_prps: + iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); + iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); + nvme_free_descriptors(req); + return BLK_STS_RESOURCE; + } + static void nvme_free_prps(struct request *req, unsigned int attrs) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); @@@ -943,6 -1015,9 +1015,9 @@@ static void nvme_unmap_data(struct requ struct device *dma_dev = nvmeq->dev->dev; unsigned int attrs = 0; + if (blk_mq_rq_is_dmabuf(req)) + goto out_free_desc; + if (iod->flags & IOD_SINGLE_SEGMENT) { static_assert(offsetof(union nvme_data_ptr, prp1) == offsetof(union nvme_data_ptr, sgl.addr)); @@@ -967,6 -1042,7 +1042,7 @@@ nvme_free_prps(req, attrs); } + out_free_desc: if (iod->nr_descriptors) nvme_free_descriptors(req); } @@@ -1148,11 -1224,17 +1224,17 @@@ bad_sgl } static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge, + dma_addr_t addr, u32 len) + { + sge->addr = cpu_to_le64(addr); + sge->length = cpu_to_le32(len); + sge->type = NVME_SGL_FMT_DATA_DESC << 4; + } + + static void nvme_pci_dma_iter_set_sgl(struct nvme_sgl_desc *sge, struct blk_dma_iter *iter) { - sge->addr = cpu_to_le64(iter->addr); - sge->length = cpu_to_le32(iter->len); - sge->type = NVME_SGL_FMT_DATA_DESC << 4; + nvme_pci_sgl_set_data(sge, iter->addr, iter->len); } static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge, @@@ -1163,6 -1245,142 +1245,142 @@@ sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; } + static unsigned int nvme_pci_dmabuf_sgl_nents(struct request *req) + { + struct bio *bio = req->bio; + struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map); + struct scatterlist *sg; + unsigned long tmp; + size_t offset = bio->bi_iter.bi_offset; + size_t remaining = blk_rq_payload_bytes(req); + unsigned int nents = 0; + + for_each_sgtable_dma_sg(map->sgt, sg, tmp) { + size_t sg_len = sg_dma_len(sg); + + if (!remaining) + break; + if (offset >= sg_len) { + offset -= sg_len; + continue; + } + + sg_len -= offset; + offset = 0; + + do { + size_t chunk = min(remaining, sg_len); + + nents++; + sg_len -= chunk; + remaining -= chunk; + } while (sg_len && remaining); + } + + if (unlikely(remaining)) + return 0; + + return nents; + } + + static blk_status_t nvme_rq_setup_dmabuf_sgl(struct request *req, + struct nvme_queue *nvmeq, unsigned int entries) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_dmabuf_map *map = nvme_req_to_map(req); + size_t length = blk_rq_payload_bytes(req); + struct nvme_sgl_desc *sg_list = &iod->cmd.common.dptr.sgl; + size_t offset = req->bio->bi_iter.bi_offset; + bool pooled = entries > 1; + dma_addr_t sgl_dma = 0; + unsigned int mapped = 0; + unsigned long tmp; + struct scatterlist *sg; + size_t remaining = length; + + if (!entries) + return BLK_STS_IOERR; + + iod->cmd.common.flags = NVME_CMD_SGL_METABUF; + iod->total_len = length; + + /* entries == 1 fits in the inline descriptor; more needs a pool. */ + if (pooled) { + if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list)) + iod->flags |= IOD_SMALL_DESCRIPTOR; + + sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, + &sgl_dma); + if (!sg_list) + return BLK_STS_RESOURCE; + iod->descriptors[iod->nr_descriptors++] = sg_list; + } + + for_each_sgtable_dma_sg(map->sgt, sg, tmp) { + size_t sg_len = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); + + if (!remaining) + break; + if (offset >= sg_len) { + offset -= sg_len; + continue; + } + + addr += offset; + sg_len -= offset; + offset = 0; + + do { + u32 chunk = min_t(size_t, remaining, sg_len); + + if (WARN_ON_ONCE(mapped == entries)) + goto err_free; + nvme_pci_sgl_set_data(&sg_list[mapped++], addr, chunk); + + addr += chunk; + sg_len -= chunk; + remaining -= chunk; + } while (sg_len && remaining); + } + + if (unlikely(remaining)) + goto err_free; + + if (pooled) + nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, + mapped); + return BLK_STS_OK; + err_free: + if (pooled) { + iod->nr_descriptors--; + dma_pool_free(nvme_dma_pool(nvmeq, iod), sg_list, sgl_dma); + } + return BLK_STS_IOERR; + } + + static blk_status_t nvme_rq_setup_dmabuf(struct request *req, + struct nvme_queue *nvmeq, enum nvme_use_sgl use_sgl) + { + unsigned int entries; + size_t avg_seg; + + if (use_sgl == SGL_UNSUPPORTED) + return nvme_rq_setup_dmabuf_map(req, nvmeq); + + entries = nvme_pci_dmabuf_sgl_nents(req); + + if (use_sgl == SGL_FORCED) + return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries); + + avg_seg = entries ? + DIV_ROUND_UP(blk_rq_payload_bytes(req), entries) : 0; + if (sgl_threshold && avg_seg >= sgl_threshold) + return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries); + + return nvme_rq_setup_dmabuf_map(req, nvmeq); + } + static blk_status_t nvme_pci_setup_data_sgl(struct request *req, struct blk_dma_iter *iter) { @@@ -1177,7 -1395,7 +1395,7 @@@ iod->cmd.common.flags = NVME_CMD_SGL_METABUF; if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { - nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); + nvme_pci_dma_iter_set_sgl(&iod->cmd.common.dptr.sgl, iter); iod->total_len += iter->len; return BLK_STS_OK; } @@@ -1199,7 -1417,7 +1417,7 @@@ iter->status = BLK_STS_IOERR; break; } - nvme_pci_sgl_set_data(&sg_list[mapped++], iter); + nvme_pci_dma_iter_set_sgl(&sg_list[mapped++], iter); iod->total_len += iter->len; } while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); @@@ -1257,6 -1475,9 +1475,9 @@@ static blk_status_t nvme_map_data(struc struct blk_dma_iter iter; blk_status_t ret; + if (blk_mq_rq_is_dmabuf(req)) + return nvme_rq_setup_dmabuf(req, nvmeq, use_sgl); + /* * Try to skip the DMA iterator for single segment requests, as that * significantly improves performances for small I/O sizes. @@@ -1359,13 -1580,13 +1580,13 @@@ static blk_status_t nvme_pci_setup_meta iod->cmd.common.metadata = cpu_to_le64(sgl_dma); if (entries == 1) { iod->meta_total_len = iter.len; - nvme_pci_sgl_set_data(sg_list, &iter); + nvme_pci_dma_iter_set_sgl(sg_list, &iter); return BLK_STS_OK; } sgl_dma += sizeof(*sg_list); do { - nvme_pci_sgl_set_data(&sg_list[++i], &iter); + nvme_pci_dma_iter_set_sgl(&sg_list[++i], &iter); iod->meta_total_len += iter.len; } while (blk_rq_integrity_dma_map_iter_next(req, dev->dev, &iter)); @@@ -3357,6 -3578,9 +3578,9 @@@ static void nvme_dev_disable(struct nvm nvme_unquiesce_admin_queue(&dev->ctrl); } mutex_unlock(&dev->shutdown_lock); + + if (shutdown) + nvme_pci_remove_dmabuf(&dev->dmabufs); } static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) @@@ -3564,6 -3788,14 +3788,14 @@@ static unsigned long nvme_pci_get_virt_ return 0; } + static int nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, + struct dma_buf_io_ctx *ctx) + { + struct nvme_dev *dev = to_nvme_dev(ctrl); + + return nvme_init_dma_buf_io_ctx(ctrl, &dev->dmabufs, ctx); + } + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .name = "pcie", .module = THIS_MODULE, @@@ -3579,6 -3811,7 +3811,7 @@@ .print_device_info = nvme_pci_print_device_info, .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma, .get_virt_boundary = nvme_pci_get_virt_boundary, + .init_dma_buf_io_ctx = nvme_pci_init_dma_buf_io_ctx, }; static int nvme_dev_map(struct nvme_dev *dev) @@@ -3701,6 -3934,7 +3934,7 @@@ static struct nvme_dev *nvme_pci_alloc_ return ERR_PTR(-ENOMEM); INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); mutex_init(&dev->shutdown_lock); + nvme_init_buf_list(&dev->dmabufs); dev->nr_write_queues = write_queues; dev->nr_poll_queues = poll_queues; @@@ -3747,7 -3981,7 +3981,7 @@@ */ dev->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_BYTES >> SECTOR_SHIFT, - dma_max_opt_mapping_size(&pdev->dev) >> SECTOR_SHIFT); + dma_max_mapping_size(&pdev->dev) >> 9); dev->ctrl.max_segments = NVME_MAX_SEGS; dev->ctrl.max_integrity_segments = 1; return dev; @@@ -4346,5 -4580,6 +4580,6 @@@ MODULE_AUTHOR("Matthew Wilcox