[PATCH 46/56] drm/i915/bdw: implement alloc/teardown for 4lvl

[Ben Widawsky <benjamin.widawsky@intel.com>]

The code for 4lvl works just as one would expect, and nicely it is able
to call into the existing 3lvl page table code to handle all of the
lower levels.

PML4 has no special attributes.

Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
---
 drivers/gpu/drm/i915/i915_gem_gtt.c | 170 ++++++++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_gem_gtt.h |  12 ++-
 2 files changed, 163 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index c4b53ef..3478bf5 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -413,9 +413,12 @@ static void __pdp_fini(struct i915_pagedirpo *pdp)
 static void free_pdp_single(struct i915_pagedirpo *pdp,
 			    struct drm_device *dev)
 {
-	__pdp_fini(pdp);
-	if (HAS_48B_PPGTT(dev))
+	if (HAS_48B_PPGTT(dev)) {
+		__pdp_fini(pdp);
+		i915_dma_unmap_single(pdp, dev);
+		__free_page(pdp->page);
 		kfree(pdp);
+	}
 }
 
 static int __pdp_init(struct i915_pagedirpo *pdp,
@@ -441,6 +444,58 @@ static int __pdp_init(struct i915_pagedirpo *pdp,
 	return 0;
 }
 
+static struct i915_pagedirpo *alloc_pdp_single(struct i915_hw_ppgtt *ppgtt,
+					       struct i915_pml4 *pml4)
+{
+	struct drm_device *dev = ppgtt->base.dev;
+	struct i915_pagedirpo *pdp;
+	int ret;
+
+	BUG_ON(!HAS_48B_PPGTT(dev));
+
+	pdp = kmalloc(sizeof(*pdp), GFP_KERNEL);
+	if (!pdp)
+		return ERR_PTR(-ENOMEM);
+
+	pdp->page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
+	if (!pdp->page) {
+		kfree(pdp);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	ret = __pdp_init(pdp, dev);
+	if (ret) {
+		__free_page(pdp->page);
+		kfree(pdp);
+		return ERR_PTR(ret);
+	}
+
+	i915_dma_map_px_single(pdp, dev);
+
+	return pdp;
+}
+
+static void pml4_fini(struct i915_pml4 *pml4)
+{
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(pml4, struct i915_hw_ppgtt, pml4);
+	i915_dma_unmap_single(pml4, ppgtt->base.dev);
+	__free_page(pml4->page);
+}
+
+static int pml4_init(struct i915_hw_ppgtt *ppgtt)
+{
+	struct i915_pml4 *pml4 = &ppgtt->pml4;
+
+	pml4->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!pml4->page)
+		return -ENOMEM;
+
+	i915_dma_map_px_single(pml4, ppgtt->base.dev);
+
+	return 0;
+}
+
 /* Broadwell Page Directory Pointer Descriptors */
 static int gen8_write_pdp(struct intel_ring_buffer *ring,
 			  unsigned entry,
@@ -729,7 +784,14 @@ static void gen8_teardown_va_range_4lvl(struct i915_address_space *vm,
 					struct i915_pml4 *pml4,
 					uint64_t start, uint64_t length)
 {
-	BUG();
+	struct i915_pagedirpo *pdp;
+	uint64_t temp, pml4e;
+
+	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+		gen8_teardown_va_range_3lvl(vm, pdp, start, length);
+		if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev)))
+			clear_bit(pml4e, pml4->used_pml4es);
+	}
 }
 
 static void gen8_teardown_va_range(struct i915_address_space *vm,
@@ -738,10 +800,10 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 
-	if (!HAS_48B_PPGTT(vm->dev))
-		gen8_teardown_va_range_3lvl(vm, &ppgtt->pdp, start, length);
-	else
+	if (HAS_48B_PPGTT(vm->dev))
 		gen8_teardown_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+	else
+		gen8_teardown_va_range_3lvl(vm, &ppgtt->pdp, start, length);
 }
 
 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
@@ -1021,12 +1083,76 @@ err_out:
 	return ret;
 }
 
-static int __noreturn gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
-					       struct i915_pml4 *pml4,
-					       uint64_t start,
-					       uint64_t length)
+static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
+				    struct i915_pml4 *pml4,
+				    uint64_t start,
+				    uint64_t length)
 {
-	BUG();
+	DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(vm, struct i915_hw_ppgtt, base);
+	struct i915_pagedirpo *pdp;
+	const uint64_t orig_start = start;
+	const uint64_t orig_length = length;
+	uint64_t temp, pml4e;
+
+	/* Do the pml4 allocations first, so we don't need to track the newly
+	 * allocated tables below the pdp */
+	bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
+
+	/* The pagedirectory and pagetable allocations are done in the shared 3
+	 * and 4 level code. Just allocate the pdps.
+	 */
+	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+		if (!pdp) {
+			WARN_ON(test_bit(pml4e, pml4->used_pml4es));
+			pdp = alloc_pdp_single(ppgtt, pml4);
+			if (IS_ERR(pdp))
+				goto err_alloc;
+
+			pml4->pdps[pml4e] = pdp;
+			set_bit(pml4e, new_pdps);
+			trace_i915_pagedirpo_alloc(&ppgtt->base, pml4e,
+						   pml4e << GEN8_PML4E_SHIFT,
+						   GEN8_PML4E_SHIFT);
+
+		} else
+			WARN(!test_bit(pml4e, pml4->used_pml4es),
+			     "%lld %p", pml4e, vm);
+	}
+
+	start = orig_start;
+	length = orig_length;
+
+	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+		int ret;
+
+		BUG_ON(!pdp);
+
+		ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
+		if (ret)
+			goto err_out;
+	}
+
+	WARN(bitmap_weight(pml4->used_pml4es, GEN8_PML4ES_PER_PML4) > 2,
+	     "The allocation has spanned more than 512GB. It is highly likely this is incorrect.");
+
+	bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
+		  GEN8_PML4ES_PER_PML4);
+
+	return 0;
+
+err_out:
+	/* This will teardown more than we allocated. It should be fine, and
+	 * makes code simpler. */
+	start = orig_start;
+	length = orig_length;
+	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e)
+		gen8_teardown_va_range_3lvl(vm, pdp, start, length);
+
+err_alloc:
+	for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
+		free_pdp_single(pdp, vm->dev);
 }
 
 static int gen8_alloc_va_range(struct i915_address_space *vm,
@@ -1035,16 +1161,19 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 
-	if (!HAS_48B_PPGTT(vm->dev))
-		return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
-	else
+	if (HAS_48B_PPGTT(vm->dev))
 		return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+	else
+		return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
 }
 
 static void gen8_ppgtt_fini_common(struct i915_hw_ppgtt *ppgtt)
 {
 	free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
-	free_pdp_single(&ppgtt->pdp, ppgtt->base.dev);
+	if (HAS_48B_PPGTT(ppgtt->base.dev))
+		pml4_fini(&ppgtt->pml4);
+	else
+		free_pdp_single(&ppgtt->pdp, ppgtt->base.dev);
 }
 
 /**
@@ -1066,7 +1195,13 @@ static int gen8_ppgtt_init_common(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	if (IS_ERR(ppgtt->scratch_pd))
 		return PTR_ERR(ppgtt->scratch_pd);
 
-	if (!HAS_48B_PPGTT(ppgtt->base.dev)) {
+	if (HAS_48B_PPGTT(ppgtt->base.dev)) {
+		int ret = pml4_init(ppgtt);
+		if (ret) {
+			free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
+			return ret;
+		}
+	} else {
 		int ret = __pdp_init(&ppgtt->pdp, false);
 		if (ret) {
 			free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
@@ -1075,8 +1210,7 @@ static int gen8_ppgtt_init_common(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 
 		ppgtt->switch_mm = gen8_mm_switch;
 		trace_i915_pagedirpo_alloc(&ppgtt->base, 0, 0, GEN8_PML4E_SHIFT);
-	} else
-		BUG(); /* Not yet implemented */
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 94c825e..0e5cd58 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -103,6 +103,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
 #endif
 #define GEN8_PML4ES_PER_PML4		512
 #define GEN8_PML4E_SHIFT		39
+#define GEN8_PML4E_MASK			(GEN8_PML4ES_PER_PML4 - 1)
 #define GEN8_PDPE_SHIFT			30
 /* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
  * tables */
@@ -436,9 +437,18 @@ static inline size_t gen6_pde_count(uint32_t addr, uint32_t length)
 	     temp = min(temp, length),					\
 	     start += temp, length -= temp)
 
+#define gen8_for_each_pml4e(pdp, pml4, start, length, temp, iter)	\
+	for (iter = gen8_pml4e_index(start), pdp = (pml4)->pdps[iter];	\
+	     length > 0 && iter < GEN8_PML4ES_PER_PML4;			\
+	     pdp = (pml4)->pdps[++iter],				\
+	     temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT) - start,	\
+	     temp = min(temp, length),					\
+	     start += temp, length -= temp)
+
 #define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)		\
 	gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, I915_PDPES_PER_PDP(dev))
 
+
 /* Clamp length to the next pagetab boundary */
 static inline uint64_t gen8_clamp_pt(uint64_t start, uint64_t length)
 {
@@ -476,7 +486,7 @@ static inline uint32_t gen8_pdpe_index(uint64_t address)
 
 static inline uint32_t gen8_pml4e_index(uint64_t address)
 {
-	BUG();
+	return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK;
 }
 
 static inline size_t gen8_pte_count(uint64_t addr, uint64_t length)
-- 
1.9.2