[PATCH 45/56] drm/i915/bdw: Abstract PDP usage

[Ben Widawsky <benjamin.widawsky@intel.com>]

Up until now, ppgtt->pdp has always been the root of our page tables.
Legacy 32b addresses acted like it had 1 PDP with 4 PDPEs.

In preparation for 4 level page tables, we need to stop use ppgtt->pdp
directly unless we know it's what we want. The future structure will use
ppgtt->pml4 for the top level, and the pdp is just one of the entries
being pointed to by a pml4e.

This patch addresses some carelessness done throughout development wrt
assumptions made of the root page tables.

Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
---
 drivers/gpu/drm/i915/i915_gem_gtt.c | 129 ++++++++++++++++++++++++------------
 1 file changed, 85 insertions(+), 44 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index df3cd41..c4b53ef 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -499,6 +499,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 {
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
+	struct i915_pagedirpo *pdp = &ppgtt->pdp; /* FIXME: 48b */
 	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
 	unsigned pdpe = gen8_pdpe_index(start);
 	unsigned pde = gen8_pde_index(start);
@@ -510,7 +511,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 				      I915_CACHE_LLC, use_scratch);
 
 	while (num_entries) {
-		struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
+		struct i915_pagedir *pd = pdp->pagedirs[pdpe];
 		struct i915_pagetab *pt = pd->page_tables[pde];
 		struct page *page_table = pt->page;
 
@@ -544,6 +545,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 {
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
+	struct i915_pagedirpo *pdp = &ppgtt->pdp; /* FIXME: 48b */
 	gen8_gtt_pte_t *pt_vaddr;
 	unsigned pdpe = gen8_pdpe_index(start);
 	unsigned pde = gen8_pde_index(start);
@@ -554,7 +556,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 		if (pt_vaddr == NULL) {
-			struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
+			struct i915_pagedir *pd = pdp->pagedirs[pdpe];
 			struct i915_pagetab *pt = pd->page_tables[pde];
 			struct page *page_table = pt->page;
 			pt_vaddr = kmap_atomic(page_table);
@@ -636,23 +638,22 @@ static void gen8_unmap_pagetable(struct i915_hw_ppgtt *ppgtt,
 	gen8_map_pagedir(pd, ppgtt->scratch_pt, pde, ppgtt->base.dev);
 }
 
-static void gen8_teardown_va_range(struct i915_address_space *vm,
-				   uint64_t start, uint64_t length)
+static void gen8_teardown_va_range_3lvl(struct i915_address_space *vm,
+					struct i915_pagedirpo *pdp,
+					uint64_t start, uint64_t length)
 {
-	struct i915_hw_ppgtt *ppgtt =
-		        container_of(vm, struct i915_hw_ppgtt, base);
 	struct drm_device *dev = vm->dev;
 	struct i915_pagedir *pd;
 	struct i915_pagetab *pt;
 	uint64_t temp;
 	uint32_t pdpe, pde, orig_start = start;
 
-	if (!ppgtt->pdp.pagedirs) {
+	if (!pdp || !pdp->pagedirs) {
 		/* If pagedirs are already free, there is nothing to do.*/
 		return;
 	}
 
-	gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
 		uint64_t pd_len = gen8_clamp_pd(start, length);
 		uint64_t pd_start = start;
 
@@ -660,12 +661,12 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,
 		 * down, and up.
 		 */
 		if (!pd) {
-			WARN(test_bit(pdpe, ppgtt->pdp.used_pdpes),
+			WARN(test_bit(pdpe, pdp->used_pdpes),
 			     "PDPE %d is not allocated, but is reserved (%p)\n",
 			     pdpe, vm);
 			continue;
 		} else {
-			WARN(!test_bit(pdpe, ppgtt->pdp.used_pdpes),
+			WARN(!test_bit(pdpe, pdp->used_pdpes),
 			     "PDPE %d not reserved, but is allocated (%p)",
 			     pdpe, vm);
 		}
@@ -691,6 +692,8 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,
 				     gen8_pte_count(pd_start, pd_len));
 
 			if (bitmap_empty(pt->used_ptes, GEN8_PTES_PER_PT)) {
+				struct i915_hw_ppgtt *ppgtt =
+					container_of(vm, struct i915_hw_ppgtt, base);
 				trace_i915_pagetable_destroy(vm,
 							     pde,
 							     pd_start & GENMASK_ULL(64, GEN8_PDE_SHIFT),
@@ -705,23 +708,42 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,
 
 		if (bitmap_empty(pd->used_pdes, I915_PDES_PER_PD)) {
 			free_pd_single(pd, dev);
-			ppgtt->pdp.pagedirs[pdpe] = NULL;
+			pdp->pagedirs[pdpe] = NULL;
 			trace_i915_pagedirectory_destroy(vm, pdpe,
 							 start & GENMASK_ULL(64, GEN8_PDPE_SHIFT),
 							 GEN8_PDPE_SHIFT);
-			WARN_ON(!test_and_clear_bit(pdpe, ppgtt->pdp.used_pdpes));
+			WARN_ON(!test_and_clear_bit(pdpe, pdp->used_pdpes));
 		}
 	}
 
-	if (bitmap_empty(ppgtt->pdp.used_pdpes, I915_PDPES_PER_PDP(dev))) {
+	if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev))) {
 		/* TODO: When pagetables are fully dynamic:
-		free_pdp_single(&ppgtt->pdp, dev); */
+		free_pdp_single(pdp, dev); */
 		trace_i915_pagedirpo_destroy(vm, 0,
 					     orig_start & GENMASK_ULL(64, GEN8_PML4E_SHIFT),
 					     GEN8_PML4E_SHIFT);
 	}
 }
 
+static void gen8_teardown_va_range_4lvl(struct i915_address_space *vm,
+					struct i915_pml4 *pml4,
+					uint64_t start, uint64_t length)
+{
+	BUG();
+}
+
+static void gen8_teardown_va_range(struct i915_address_space *vm,
+				   uint64_t start, uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(vm, struct i915_hw_ppgtt, base);
+
+	if (!HAS_48B_PPGTT(vm->dev))
+		gen8_teardown_va_range_3lvl(vm, &ppgtt->pdp, start, length);
+	else
+		gen8_teardown_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+}
+
 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 {
 	trace_i915_va_teardown(&ppgtt->base,
@@ -747,7 +769,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 
 /**
  * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
- * @ppgtt:	Master ppgtt structure.
+ * @vm:		Master vm structure.
  * @pd:		Page directory for this address range.
  * @start:	Starting virtual address to begin allocations.
  * @length	Size of the allocations.
@@ -763,13 +785,13 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
  *
  * Return: 0 if success; negative error code otherwise.
  */
-static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt,
+static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
 				     struct i915_pagedir *pd,
 				     uint64_t start,
 				     uint64_t length,
 				     unsigned long *new_pts)
 {
-	struct drm_device *dev = ppgtt->base.dev;
+	struct drm_device *dev = vm->dev;
 	struct i915_pagetab *unused;
 	uint64_t temp;
 	uint32_t pde;
@@ -784,7 +806,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt,
 			goto unwind_out;
 
 		set_bit(pde, new_pts);
-		trace_i915_pagetable_alloc(&ppgtt->base, pde, start, GEN8_PDE_SHIFT);
+		trace_i915_pagetable_alloc(vm, pde, start, GEN8_PDE_SHIFT);
 	}
 
 	return 0;
@@ -798,7 +820,7 @@ unwind_out:
 
 /**
  * gen8_ppgtt_alloc_pagedirs() - Allocate page directories for VA range.
- * @ppgtt:	Master ppgtt structure.
+ * @vm:		Master vm structure.
  * @pdp:	Page directory pointer for this address range.
  * @start:	Starting virtual address to begin allocations.
  * @length	Size of the allocations.
@@ -819,17 +841,17 @@ unwind_out:
  *
  * Return: 0 if success; negative error code otherwise.
  */
-static int gen8_ppgtt_alloc_pagedirs(struct i915_hw_ppgtt *ppgtt,
+static int gen8_ppgtt_alloc_pagedirs(struct i915_address_space *vm,
 				     struct i915_pagedirpo *pdp,
 				     uint64_t start,
 				     uint64_t length,
 				     unsigned long *new_pds)
 {
-	struct drm_device *dev = ppgtt->base.dev;
+	struct drm_device *dev = vm->dev;
 	struct i915_pagedir *unused;
 	uint64_t temp;
 	uint32_t pdpe;
-	size_t pdpes =  I915_PDPES_PER_PDP(ppgtt->base.dev);
+	size_t pdpes =  I915_PDPES_PER_PDP(vm->dev);
 
 	BUG_ON(!bitmap_empty(new_pds, pdpes));
 
@@ -841,13 +863,13 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_hw_ppgtt *ppgtt,
 		if (unused)
 			continue;
 
-		pd = alloc_pd_single(ppgtt->base.dev);
+		pd = alloc_pd_single(dev);
 		if (IS_ERR(pd))
 			goto unwind_out;
 
 		pdp->pagedirs[pdpe] = pd;
 		set_bit(pdpe, new_pds);
-		trace_i915_pagedirectory_alloc(&ppgtt->base, pdpe, start,
+		trace_i915_pagedirectory_alloc(vm, pdpe, start,
 					       GEN8_PDPE_SHIFT);
 	}
 
@@ -855,7 +877,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_hw_ppgtt *ppgtt,
 
 unwind_out:
 	for_each_set_bit(pdpe, new_pds, pdpes)
-		free_pd_single(pdp->pagedirs[pdpe], ppgtt->base.dev);
+		free_pd_single(pdp->pagedirs[pdpe], dev);
 
 	return -ENOMEM;
 }
@@ -910,12 +932,11 @@ err_out:
 	return -ENOMEM;
 }
 
-static int gen8_alloc_va_range(struct i915_address_space *vm,
-			       uint64_t start,
-			       uint64_t length)
+static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
+				    struct i915_pagedirpo *pdp,
+				    uint64_t start,
+				    uint64_t length)
 {
-	struct i915_hw_ppgtt *ppgtt =
-		container_of(vm, struct i915_hw_ppgtt, base);
 	unsigned long *new_page_dirs, **new_page_tables;
 	struct drm_device *dev = vm->dev;
 	struct i915_pagedir *pd;
@@ -945,17 +966,15 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 		return ret;
 
 	/* Do the allocations first so we can easily bail out */
-	ret = gen8_ppgtt_alloc_pagedirs(ppgtt, &ppgtt->pdp, start, length,
-					new_page_dirs);
+	ret = gen8_ppgtt_alloc_pagedirs(vm, pdp, start, length, new_page_dirs);
 	if (ret) {
 		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
 		return ret;
 	}
 
-	gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
 		bitmap_zero(new_page_tables[pdpe], I915_PDES_PER_PD);
-		ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length,
-						new_page_tables[pdpe]);
+		ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, new_page_tables[pdpe]);
 		if (ret)
 			goto err_out;
 	}
@@ -963,7 +982,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 	start = orig_start;
 	length = orig_length;
 
-	gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
 		struct i915_pagetab *pt;
 		uint64_t pd_len = gen8_clamp_pd(start, length);
 		uint64_t pd_start = start;
@@ -981,7 +1000,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 			set_bit(pde, pd->used_pdes);
 		}
 
-		set_bit(pdpe, ppgtt->pdp.used_pdpes);
+		set_bit(pdpe, pdp->used_pdpes);
 
 		gen8_map_pagetable_range(vm, pd, start, length);
 	}
@@ -992,16 +1011,36 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 err_out:
 	while (pdpe--) {
 		for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES_PER_PD)
-			free_pt_single(pd->page_tables[temp], ppgtt->base.dev);
+			free_pt_single(pd->page_tables[temp], dev);
 	}
 
 	for_each_set_bit(pdpe, new_page_dirs, pdpes)
-		free_pd_single(ppgtt->pdp.pagedirs[pdpe], ppgtt->base.dev);
+		free_pd_single(pdp->pagedirs[pdpe], dev);
 
 	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
 	return ret;
 }
 
+static int __noreturn gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
+					       struct i915_pml4 *pml4,
+					       uint64_t start,
+					       uint64_t length)
+{
+	BUG();
+}
+
+static int gen8_alloc_va_range(struct i915_address_space *vm,
+			       uint64_t start, uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(vm, struct i915_hw_ppgtt, base);
+
+	if (!HAS_48B_PPGTT(vm->dev))
+		return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
+	else
+		return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+}
+
 static void gen8_ppgtt_fini_common(struct i915_hw_ppgtt *ppgtt)
 {
 	free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
@@ -1046,12 +1085,13 @@ static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 {
 	struct drm_device *dev = ppgtt->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct i915_pagedirpo *pdp = &ppgtt->pdp; /* FIXME: 48b */
 	struct i915_pagedir *pd;
 	uint64_t temp, start = 0, size = dev_priv->gtt.base.total;
 	uint32_t pdpe;
 	int ret;
 
-	ret = gen8_ppgtt_init_common(ppgtt, dev_priv->gtt.base.total);
+	ret = gen8_ppgtt_init_common(ppgtt, size);
 	if (ret)
 		return ret;
 
@@ -1061,7 +1101,7 @@ static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 		return ret;
 	}
 
-	gen8_for_each_pdpe(pd, &ppgtt->pdp, start, size, temp, pdpe)
+	gen8_for_each_pdpe(pd, pdp, start, size, temp, pdpe)
 		gen8_map_pagetable_range(&ppgtt->base, pd, start, size);
 
 	ppgtt->base.allocate_va_range = NULL;
@@ -2101,6 +2141,7 @@ void gen8_for_every_pdpe_pde(struct i915_hw_ppgtt *ppgtt,
 			     void *data)
 {
 	struct drm_device *dev = ppgtt->base.dev;
+	struct i915_pagedirpo *pdp = &ppgtt->pdp; /* FIXME: 48b */
 	uint64_t start = ppgtt->base.start;
 	uint64_t length = ppgtt->base.total;
 	uint64_t pdpe, pde, temp;
@@ -2108,18 +2149,18 @@ void gen8_for_every_pdpe_pde(struct i915_hw_ppgtt *ppgtt,
 	struct i915_pagedir *pd;
 	struct i915_pagetab *pt;
 
-	gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
 		uint64_t pd_start = start, pd_length = length;
 		int i;
 
 		if (pd == NULL) {
 			for (i = 0; i < I915_PDES_PER_PD; i++)
-				callback(&ppgtt->pdp, NULL, NULL, pdpe, i, data);
+				callback(pdp, NULL, NULL, pdpe, i, data);
 			continue;
 		}
 
 		gen8_for_each_pde(pt, pd, pd_start, pd_length, temp, pde) {
-			callback(&ppgtt->pdp, pd, pt, pdpe, pde, data);
+			callback(pdp, pd, pt, pdpe, pde, data);
 		}
 	}
 }
-- 
1.9.2