[PATCH v6 07/12] mm/cma: Introduce cma_alloc_at() API

[Thierry Reding <thierry.reding@kernel.org>]

From: Thierry Reding <treding@nvidia.com>

This API can be used to allocate a number of CMA pages starting at a
fixed offset. This is useful, for example, if the CMA area is used as
backing storage for a nested allocator that has stricter requirements
than CMA itself.

In order to reuse most of the existing code, the cma_range_alloc()
function is enhanced to take as inputs a range of pages to look for a
free region. If this range encompasses the entire CMA range, the
behavior is as before, but the range can also be specified to match
the offset and size passed in from cma_alloc_at(), in which case that
is the range that will be allocated (or allocation will fail if the
range is not available).

Suggested-by: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: Thierry Reding <treding@nvidia.com>
---
Changes in v6:
- refactor cma_range_alloc() so it can be reused for _at() code paths
---
 include/linux/cma.h        |  4 +++
 include/trace/events/cma.h | 77 ++++++++++++++++++++++++++++++++++++++--
 mm/cma.c                   | 88 +++++++++++++++++++++++++++++++++++-----------
 3 files changed, 146 insertions(+), 23 deletions(-)

diff --git a/include/linux/cma.h b/include/linux/cma.h
index 8555d38a97b1..844404459a42 100644
--- a/include/linux/cma.h
+++ b/include/linux/cma.h
@@ -49,11 +49,15 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
 					struct cma **res_cma);
 extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
 			      bool no_warn);
+extern struct page *cma_alloc_at(struct cma *cma, unsigned long offset,
+				 unsigned long count, bool no_warn);
 extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
 struct page *cma_alloc_frozen(struct cma *cma, unsigned long count,
 		unsigned int align, bool no_warn);
 struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order);
+struct page *cma_alloc_at_frozen(struct cma *cma, unsigned long offset,
+				 unsigned long count, bool no_warn);
 bool cma_release_frozen(struct cma *cma, const struct page *pages,
 		unsigned long count);
 
diff --git a/include/trace/events/cma.h b/include/trace/events/cma.h
index 37195edf2498..4e0ab63e303f 100644
--- a/include/trace/events/cma.h
+++ b/include/trace/events/cma.h
@@ -38,13 +38,17 @@ TRACE_EVENT(cma_release,
 
 TRACE_EVENT(cma_alloc_start,
 
-	TP_PROTO(const char *name, unsigned long request_count, unsigned long available_count,
+	TP_PROTO(const char *name, unsigned long start, unsigned long end,
+		unsigned long request_count, unsigned long available_count,
 		unsigned long total_count, unsigned int align),
 
-	TP_ARGS(name, request_count, available_count, total_count, align),
+	TP_ARGS(name, start, end, request_count, available_count, total_count,
+		align),
 
 	TP_STRUCT__entry(
 		__string(name, name)
+		__field(unsigned long, start)
+		__field(unsigned long, end)
 		__field(unsigned long, request_count)
 		__field(unsigned long, available_count)
 		__field(unsigned long, total_count)
@@ -53,14 +57,18 @@ TRACE_EVENT(cma_alloc_start,
 
 	TP_fast_assign(
 		__assign_str(name);
+		__entry->start = start;
+		__entry->end = end;
 		__entry->request_count = request_count;
 		__entry->available_count = available_count;
 		__entry->total_count = total_count;
 		__entry->align = align;
 	),
 
-	TP_printk("name=%s request_count=%lu available_count=%lu total_count=%lu align=%u",
+	TP_printk("name=%s start=%lu end=%lu request_count=%lu available_count=%lu total_count=%lu align=%u",
 		  __get_str(name),
+		  __entry->start,
+		  __entry->end,
 		  __entry->request_count,
 		  __entry->available_count,
 		  __entry->total_count,
@@ -132,6 +140,69 @@ TRACE_EVENT(cma_alloc_busy_retry,
 		  __entry->align)
 );
 
+TRACE_EVENT(cma_alloc_at_start,
+
+	TP_PROTO(const char *name, unsigned long pfn,
+		 unsigned long request_count, unsigned long available_count,
+		 unsigned long total_count),
+
+	TP_ARGS(name, pfn, request_count, available_count, total_count),
+
+	TP_STRUCT__entry(
+		__string(name, name)
+		__field(unsigned long, pfn)
+		__field(unsigned long, request_count)
+		__field(unsigned long, available_count)
+		__field(unsigned long, total_count)
+	),
+
+	TP_fast_assign(
+		__assign_str(name);
+		__entry->pfn = pfn;
+		__entry->request_count = request_count;
+		__entry->available_count = available_count;
+		__entry->total_count = total_count;
+	),
+
+	TP_printk("name=%s pfn=%lx, request_count=%lu available_count=%lu total_count=%lu",
+		  __get_str(name),
+		  __entry->pfn,
+		  __entry->request_count,
+		  __entry->available_count,
+		  __entry->total_count)
+);
+
+TRACE_EVENT(cma_alloc_at_finish,
+
+	TP_PROTO(const char *name, unsigned long pfn, const struct page *page,
+		 unsigned long count, int errorno),
+
+	TP_ARGS(name, pfn, page, count, errorno),
+
+	TP_STRUCT__entry(
+		__string(name, name)
+		__field(unsigned long, pfn)
+		__field(const struct page *, page)
+		__field(unsigned long, count)
+		__field(int, errorno)
+	),
+
+	TP_fast_assign(
+		__assign_str(name);
+		__entry->pfn = pfn;
+		__entry->page = page;
+		__entry->count = count;
+		__entry->errorno = errorno;
+	),
+
+	TP_printk("name=%s pfn=0x%lx page=%p count=%lu errorno=%d",
+		  __get_str(name),
+		  __entry->pfn,
+		  __entry->page,
+		  __entry->count,
+		  __entry->errorno)
+);
+
 #endif /* _TRACE_CMA_H */
 
 /* This part must be outside protection */
diff --git a/mm/cma.c b/mm/cma.c
index a10ea37a261d..f92d3a188209 100644
--- a/mm/cma.c
+++ b/mm/cma.c
@@ -783,25 +783,32 @@ static void cma_debug_show_areas(struct cma *cma)
 	spin_unlock_irq(&cma->lock);
 }
 
+/*
+ * Searches the CMA memrange, from @start to @end, for a free region of
+ * @count bits. If @end is less or equal to @start, will search the entire
+ * memrange.
+ */
 static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr,
-				unsigned long count, unsigned int align,
-				struct page **pagep, gfp_t gfp)
+			   unsigned long start, unsigned long end,
+			   unsigned long count, unsigned int align,
+			   struct page **pagep, gfp_t gfp)
 {
 	unsigned long bitmap_maxno, bitmap_no, bitmap_count;
-	unsigned long start, pfn, mask, offset;
-	int ret = -EBUSY;
+	unsigned long pfn, mask, offset;
 	struct page *page = NULL;
+	int ret = -EBUSY;
 
 	mask = cma_bitmap_aligned_mask(cma, align);
 	offset = cma_bitmap_aligned_offset(cma, cmr, align);
-	bitmap_maxno = cma_bitmap_maxno(cma, cmr);
+	bitmap_maxno = (end > start) ? end : cma_bitmap_maxno(cma, cmr);
 	bitmap_count = cma_bitmap_pages_to_bits(cma, count);
 
 	if (bitmap_count > bitmap_maxno)
 		goto out;
 
-	for (start = 0; ; start = bitmap_no + mask + 1) {
+	while (true) {
 		spin_lock_irq(&cma->lock);
+
 		/*
 		 * If the request is larger than the available number
 		 * of pages, stop right away.
@@ -810,6 +817,7 @@ static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr,
 			spin_unlock_irq(&cma->lock);
 			break;
 		}
+
 		bitmap_no = bitmap_find_next_zero_area_off(cmr->bitmap,
 				bitmap_maxno, start, bitmap_count, mask,
 				offset);
@@ -835,10 +843,11 @@ static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr,
 
 		bitmap_set(cmr->bitmap, bitmap_no, bitmap_count);
 		cma->available_count -= count;
+
 		/*
-		 * It's safe to drop the lock here. We've marked this region for
-		 * our exclusive use. If the migration fails we will take the
-		 * lock again and unmark it.
+		 * It's safe to drop the lock here. We've marked this region
+		 * for our exclusive use. If the migration fails we will take
+		 * the lock again and unmark it.
 		 */
 		spin_unlock_irq(&cma->lock);
 
@@ -856,37 +865,45 @@ static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr,
 			 __func__, pfn, page);
 
 		trace_cma_alloc_busy_retry(cma->name, pfn, page, count, align);
+		start = bitmap_no + mask + 1;
 	}
+
 out:
 	if (!ret)
 		*pagep = page;
+
 	return ret;
 }
 
-static struct page *__cma_alloc_frozen(struct cma *cma,
-		unsigned long count, unsigned int align, gfp_t gfp)
+static struct page *__cma_alloc_frozen(struct cma *cma, unsigned long start,
+				       unsigned long end, unsigned long count,
+				       unsigned int align, gfp_t gfp)
 {
 	struct page *page = NULL;
 	int ret = -ENOMEM, r;
 	unsigned long i;
-	const char *name = cma ? cma->name : NULL;
+
+	/* cma_alloc_at() and friends will only work with single-range CMA */
+	if (WARN_ON_ONCE(start > 0 && cma->nranges != 1))
+		return page;
 
 	if (!cma || !cma->count)
 		return page;
 
-	pr_debug("%s(cma %p, name: %s, count %lu, align %d)\n", __func__,
-		(void *)cma, cma->name, count, align);
+	pr_debug("%s(cma %p, name: %s, start %lu, end %lu, count %lu, align %d)\n",
+		 __func__, (void *)cma, cma->name, start, end, count, align);
 
 	if (!count)
 		return page;
 
-	trace_cma_alloc_start(name, count, cma->available_count, cma->count, align);
+	trace_cma_alloc_start(cma->name, start, end, count,
+			      cma->available_count, cma->count, align);
 
 	for (r = 0; r < cma->nranges; r++) {
 		page = NULL;
 
-		ret = cma_range_alloc(cma, &cma->ranges[r], count, align,
-				       &page, gfp);
+		ret = cma_range_alloc(cma, &cma->ranges[r], start, end,
+				      count, align, &page, gfp);
 		if (ret != -EBUSY || page)
 			break;
 	}
@@ -908,7 +925,7 @@ static struct page *__cma_alloc_frozen(struct cma *cma,
 	}
 
 	pr_debug("%s(): returned %p\n", __func__, page);
-	trace_cma_alloc_finish(name, page ? page_to_pfn(page) : 0,
+	trace_cma_alloc_finish(cma->name, page ? page_to_pfn(page) : 0,
 			       page, count, align, ret);
 	if (page) {
 		count_vm_event(CMA_ALLOC_SUCCESS);
@@ -926,14 +943,22 @@ struct page *cma_alloc_frozen(struct cma *cma, unsigned long count,
 {
 	gfp_t gfp = GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0);
 
-	return __cma_alloc_frozen(cma, count, align, gfp);
+	return __cma_alloc_frozen(cma, 0, 0, count, align, gfp);
 }
 
 struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order)
 {
 	gfp_t gfp = GFP_KERNEL | __GFP_COMP | __GFP_NOWARN;
 
-	return __cma_alloc_frozen(cma, 1 << order, order, gfp);
+	return __cma_alloc_frozen(cma, 0, 0, 1 << order, order, gfp);
+}
+
+struct page *cma_alloc_at_frozen(struct cma *cma, unsigned long offset,
+				 unsigned long count, bool no_warn)
+{
+	gfp_t gfp = GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0);
+
+	return __cma_alloc_frozen(cma, offset, offset + count, count, 0, gfp);
 }
 
 /**
@@ -959,6 +984,29 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 }
 EXPORT_SYMBOL_GPL(cma_alloc);
 
+/**
+ * cma_alloc_at() - allocate pages from contiguous area at fixed offset
+ * @cma:     Contiguous memory region for which the allocation is performed.
+ * @offset:  Index of the first page to allocate.
+ * @count:   Requested number of pages (in PAGE_SIZE order).
+ * @no_warn: Avoid printing message about failed allocation.
+ *
+ * This function allocates a part of the contiguous memory on a specific
+ * contiguous memory area.
+ */
+struct page *cma_alloc_at(struct cma *cma, unsigned long offset,
+			  unsigned long count, bool no_warn)
+{
+	struct page *page;
+
+	page = cma_alloc_at_frozen(cma, offset, count, no_warn);
+	if (page)
+		set_pages_refcounted(page, count);
+
+	return page;
+}
+EXPORT_SYMBOL_GPL(cma_alloc_at);
+
 static struct cma_memrange *find_cma_memrange(struct cma *cma,
 		const struct page *pages, unsigned long count)
 {

-- 
2.55.0