* [PATCH] habanalabs: refactor mmu va_range db structure
2020-11-08 16:51 [PATCH] habanalabs: improve hard reset procedure Oded Gabbay
@ 2020-11-08 16:51 ` Oded Gabbay
2020-11-08 16:51 ` [PATCH] habanalabs: Rename hw_queues_mirror to cs_mirror Oded Gabbay
1 sibling, 0 replies; 3+ messages in thread
From: Oded Gabbay @ 2020-11-08 16:51 UTC (permalink / raw)
To: linux-kernel; +Cc: SW_Drivers, Ofir Bitton
From: Ofir Bitton <obitton@habana.ai>
Use an array of va_ranges instead of keeping each va_range separately,
we do this for better readability and in order to support access to
a specific range in a much elegant manner.
Signed-off-by: Ofir Bitton <obitton@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
---
drivers/misc/habanalabs/common/habanalabs.h | 24 ++--
drivers/misc/habanalabs/common/memory.c | 130 ++++++++++----------
2 files changed, 82 insertions(+), 72 deletions(-)
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 42988f12fb00..40b566c4b791 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -953,18 +953,33 @@ struct hl_asic_funcs {
#define HL_KERNEL_ASID_ID 0
+/**
+ * enum hl_va_range_type - virtual address range type.
+ * @HL_VA_RANGE_TYPE_HOST: range type of host pages
+ * @HL_VA_RANGE_TYPE_HOST_HUGE: range type of host huge pages
+ * @HL_VA_RANGE_TYPE_DRAM: range type of dram pages
+ */
+enum hl_va_range_type {
+ HL_VA_RANGE_TYPE_HOST,
+ HL_VA_RANGE_TYPE_HOST_HUGE,
+ HL_VA_RANGE_TYPE_DRAM,
+ HL_VA_RANGE_TYPE_MAX
+};
+
/**
* struct hl_va_range - virtual addresses range.
* @lock: protects the virtual addresses list.
* @list: list of virtual addresses blocks available for mappings.
* @start_addr: range start address.
* @end_addr: range end address.
+ * @page_size: page size of this va range.
*/
struct hl_va_range {
struct mutex lock;
struct list_head list;
u64 start_addr;
u64 end_addr;
+ u32 page_size;
};
/**
@@ -993,10 +1008,7 @@ struct hl_cs_counters_atomic {
* @refcount: reference counter for the context. Context is released only when
* this hits 0l. It is incremented on CS and CS_WAIT.
* @cs_pending: array of hl fence objects representing pending CS.
- * @host_va_range: holds available virtual addresses for host mappings.
- * @host_huge_va_range: holds available virtual addresses for host mappings
- * with huge pages.
- * @dram_va_range: holds available virtual addresses for DRAM mappings.
+ * @va_range: holds available virtual addresses for host and dram mappings.
* @mem_hash_lock: protects the mem_hash.
* @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
* MMU hash or walking the PGT requires talking this lock.
@@ -1028,9 +1040,7 @@ struct hl_ctx {
struct hl_device *hdev;
struct kref refcount;
struct hl_fence **cs_pending;
- struct hl_va_range *host_va_range;
- struct hl_va_range *host_huge_va_range;
- struct hl_va_range *dram_va_range;
+ struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX];
struct mutex mem_hash_lock;
struct mutex mmu_lock;
struct list_head debugfs_list;
diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c
index e00ad11dc5f7..02233899336f 100644
--- a/drivers/misc/habanalabs/common/memory.c
+++ b/drivers/misc/habanalabs/common/memory.c
@@ -908,7 +908,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
/* get required alignment */
if (phys_pg_pack->page_size == page_size) {
- va_range = ctx->host_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
/*
* huge page alignment may be needed in case of regular
@@ -923,7 +923,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
* huge page alignment is needed in case of huge page
* mapping
*/
- va_range = ctx->host_huge_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
va_block_align = huge_page_size;
}
} else {
@@ -948,7 +948,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
hint_addr = args->map_device.hint_addr;
/* DRAM VA alignment is the same as the DRAM page size */
- va_range = ctx->dram_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
va_block_align = hdev->asic_prop.dmmu.page_size;
}
@@ -1093,12 +1093,12 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
if (phys_pg_pack->page_size ==
hdev->asic_prop.pmmu.page_size)
- va_range = ctx->host_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
else
- va_range = ctx->host_huge_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
} else if (*vm_type == VM_TYPE_PHYS_PACK) {
is_userptr = false;
- va_range = ctx->dram_va_range;
+ va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
phys_pg_pack = hnode->ptr;
} else {
dev_warn(hdev->dev,
@@ -1556,7 +1556,7 @@ bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
* addresses.
*/
static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
- u64 start, u64 end)
+ u64 start, u64 end, u32 page_size)
{
int rc;
@@ -1586,6 +1586,7 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
va_range->start_addr = start;
va_range->end_addr = end;
+ va_range->page_size = page_size;
return 0;
}
@@ -1598,8 +1599,7 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
* This function does the following:
* - Frees the virtual addresses block list and its lock
*/
-static void va_range_fini(struct hl_device *hdev,
- struct hl_va_range *va_range)
+static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range)
{
mutex_lock(&va_range->lock);
clear_va_list_locked(hdev, &va_range->list);
@@ -1629,101 +1629,96 @@ static void va_range_fini(struct hl_device *hdev,
static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
u64 host_range_start,
u64 host_range_end,
+ u32 host_page_size,
u64 host_huge_range_start,
u64 host_huge_range_end,
+ u32 host_huge_page_size,
u64 dram_range_start,
- u64 dram_range_end)
+ u64 dram_range_end,
+ u32 dram_page_size)
{
struct hl_device *hdev = ctx->hdev;
- int rc;
-
- ctx->host_va_range = kzalloc(sizeof(*ctx->host_va_range), GFP_KERNEL);
- if (!ctx->host_va_range)
- return -ENOMEM;
-
- ctx->host_huge_va_range = kzalloc(sizeof(*ctx->host_huge_va_range),
- GFP_KERNEL);
- if (!ctx->host_huge_va_range) {
- rc = -ENOMEM;
- goto host_huge_va_range_err;
- }
-
- ctx->dram_va_range = kzalloc(sizeof(*ctx->dram_va_range), GFP_KERNEL);
- if (!ctx->dram_va_range) {
- rc = -ENOMEM;
- goto dram_va_range_err;
+ int i, rc;
+
+ for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) {
+ ctx->va_range[i] =
+ kzalloc(sizeof(struct hl_va_range), GFP_KERNEL);
+ if (!ctx->va_range[i]) {
+ rc = -ENOMEM;
+ goto free_va_range;
+ }
}
rc = hl_mmu_ctx_init(ctx);
if (rc) {
dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
- goto mmu_ctx_err;
+ goto free_va_range;
}
mutex_init(&ctx->mem_hash_lock);
hash_init(ctx->mem_hash);
- mutex_init(&ctx->host_va_range->lock);
+ mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
- rc = va_range_init(hdev, ctx->host_va_range, host_range_start,
- host_range_end);
+ rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST],
+ host_range_start, host_range_end, host_page_size);
if (rc) {
dev_err(hdev->dev, "failed to init host vm range\n");
- goto host_page_range_err;
+ goto mmu_ctx_fini;
}
if (hdev->pmmu_huge_range) {
- mutex_init(&ctx->host_huge_va_range->lock);
+ mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
- rc = va_range_init(hdev, ctx->host_huge_va_range,
- host_huge_range_start,
- host_huge_range_end);
+ rc = va_range_init(hdev,
+ ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE],
+ host_huge_range_start, host_huge_range_end,
+ host_huge_page_size);
if (rc) {
dev_err(hdev->dev,
"failed to init host huge vm range\n");
- goto host_hpage_range_err;
+ goto clear_host_va_range;
}
} else {
- ctx->host_huge_va_range = ctx->host_va_range;
+ ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] =
+ ctx->va_range[HL_VA_RANGE_TYPE_HOST];
}
- mutex_init(&ctx->dram_va_range->lock);
+ mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
- rc = va_range_init(hdev, ctx->dram_va_range, dram_range_start,
- dram_range_end);
+ rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM],
+ dram_range_start, dram_range_end, dram_page_size);
if (rc) {
dev_err(hdev->dev, "failed to init dram vm range\n");
- goto dram_vm_err;
+ goto clear_host_huge_va_range;
}
hl_debugfs_add_ctx_mem_hash(hdev, ctx);
return 0;
-dram_vm_err:
- mutex_destroy(&ctx->dram_va_range->lock);
+clear_host_huge_va_range:
+ mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
if (hdev->pmmu_huge_range) {
- mutex_lock(&ctx->host_huge_va_range->lock);
- clear_va_list_locked(hdev, &ctx->host_huge_va_range->list);
- mutex_unlock(&ctx->host_huge_va_range->lock);
+ mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
+ clear_va_list_locked(hdev,
+ &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list);
+ mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
}
-host_hpage_range_err:
+clear_host_va_range:
if (hdev->pmmu_huge_range)
- mutex_destroy(&ctx->host_huge_va_range->lock);
- mutex_lock(&ctx->host_va_range->lock);
- clear_va_list_locked(hdev, &ctx->host_va_range->list);
- mutex_unlock(&ctx->host_va_range->lock);
-host_page_range_err:
- mutex_destroy(&ctx->host_va_range->lock);
+ mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
+ mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
+ clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list);
+ mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
+mmu_ctx_fini:
+ mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
mutex_destroy(&ctx->mem_hash_lock);
hl_mmu_ctx_fini(ctx);
-mmu_ctx_err:
- kfree(ctx->dram_va_range);
-dram_va_range_err:
- kfree(ctx->host_huge_va_range);
-host_huge_va_range_err:
- kfree(ctx->host_va_range);
+free_va_range:
+ for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++)
+ kfree(ctx->va_range[i]);
return rc;
}
@@ -1733,6 +1728,7 @@ int hl_vm_ctx_init(struct hl_ctx *ctx)
struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
u64 host_range_start, host_range_end, host_huge_range_start,
host_huge_range_end, dram_range_start, dram_range_end;
+ u32 host_page_size, host_huge_page_size, dram_page_size;
atomic64_set(&ctx->dram_phys_mem, 0);
@@ -1748,14 +1744,18 @@ int hl_vm_ctx_init(struct hl_ctx *ctx)
dram_range_start = prop->dmmu.start_addr;
dram_range_end = prop->dmmu.end_addr;
+ dram_page_size = prop->dmmu.page_size;
host_range_start = prop->pmmu.start_addr;
host_range_end = prop->pmmu.end_addr;
+ host_page_size = prop->pmmu.page_size;
host_huge_range_start = prop->pmmu_huge.start_addr;
host_huge_range_end = prop->pmmu_huge.end_addr;
+ host_huge_page_size = prop->pmmu_huge.page_size;
return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
- host_huge_range_start, host_huge_range_end,
- dram_range_start, dram_range_end);
+ host_page_size, host_huge_range_start,
+ host_huge_range_end, host_huge_page_size,
+ dram_range_start, dram_range_end, dram_page_size);
}
/*
@@ -1824,10 +1824,10 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx)
}
spin_unlock(&vm->idr_lock);
- va_range_fini(hdev, ctx->dram_va_range);
+ va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]);
if (hdev->pmmu_huge_range)
- va_range_fini(hdev, ctx->host_huge_va_range);
- va_range_fini(hdev, ctx->host_va_range);
+ va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
+ va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]);
mutex_destroy(&ctx->mem_hash_lock);
hl_mmu_ctx_fini(ctx);
--
2.17.1
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH] habanalabs: Rename hw_queues_mirror to cs_mirror
2020-11-08 16:51 [PATCH] habanalabs: improve hard reset procedure Oded Gabbay
2020-11-08 16:51 ` [PATCH] habanalabs: refactor mmu va_range db structure Oded Gabbay
@ 2020-11-08 16:51 ` Oded Gabbay
1 sibling, 0 replies; 3+ messages in thread
From: Oded Gabbay @ 2020-11-08 16:51 UTC (permalink / raw)
To: linux-kernel; +Cc: SW_Drivers, Tomer Tayar
From: Tomer Tayar <ttayar@habana.ai>
Future command submission types might be submitted to HW not via the
QMAN queues path. However, it would be still required to have the TDR
mechanism for these CS, and thus the patch renames the TDR fields and
replaces the hw_queues_ prefix with cs_.
Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
---
.../habanalabs/common/command_submission.c | 18 ++++++++----------
drivers/misc/habanalabs/common/device.c | 4 ++--
drivers/misc/habanalabs/common/habanalabs.h | 8 ++++----
drivers/misc/habanalabs/common/hw_queue.c | 12 ++++++------
4 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c
index ec014ef39484..516cbaacc03e 100644
--- a/drivers/misc/habanalabs/common/command_submission.c
+++ b/drivers/misc/habanalabs/common/command_submission.c
@@ -343,25 +343,24 @@ static void cs_do_release(struct kref *ref)
/* Need to update CI for internal queues */
hl_int_hw_queue_update_ci(cs);
- spin_lock(&hdev->hw_queues_mirror_lock);
- /* remove CS from hw_queues mirror list */
+ /* remove CS from CS mirror list */
+ spin_lock(&hdev->cs_mirror_lock);
list_del_init(&cs->mirror_node);
- spin_unlock(&hdev->hw_queues_mirror_lock);
+ spin_unlock(&hdev->cs_mirror_lock);
/* Don't cancel TDR in case this CS was timedout because we might be
* running from the TDR context
*/
- if (!cs->timedout &&
- hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) {
+ if (!cs->timedout && hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) {
struct hl_cs *next;
if (cs->tdr_active)
cancel_delayed_work_sync(&cs->work_tdr);
- spin_lock(&hdev->hw_queues_mirror_lock);
+ spin_lock(&hdev->cs_mirror_lock);
/* queue TDR for next CS */
- next = list_first_entry_or_null(&hdev->hw_queues_mirror_list,
+ next = list_first_entry_or_null(&hdev->cs_mirror_list,
struct hl_cs, mirror_node);
if (next && !next->tdr_active) {
@@ -370,7 +369,7 @@ static void cs_do_release(struct kref *ref)
hdev->timeout_jiffies);
}
- spin_unlock(&hdev->hw_queues_mirror_lock);
+ spin_unlock(&hdev->cs_mirror_lock);
}
out:
@@ -534,8 +533,7 @@ void hl_cs_rollback_all(struct hl_device *hdev)
flush_workqueue(hdev->cq_wq[i]);
/* Make sure we don't have leftovers in the H/W queues mirror list */
- list_for_each_entry_safe(cs, tmp, &hdev->hw_queues_mirror_list,
- mirror_node) {
+ list_for_each_entry_safe(cs, tmp, &hdev->cs_mirror_list, mirror_node) {
cs_get(cs);
cs->aborted = true;
dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n",
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
index 348faf31668b..ce0a1270e5ff 100644
--- a/drivers/misc/habanalabs/common/device.c
+++ b/drivers/misc/habanalabs/common/device.c
@@ -343,8 +343,8 @@ static int device_early_init(struct hl_device *hdev)
mutex_init(&hdev->send_cpu_message_lock);
mutex_init(&hdev->debug_lock);
mutex_init(&hdev->mmu_cache_lock);
- INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
- spin_lock_init(&hdev->hw_queues_mirror_lock);
+ INIT_LIST_HEAD(&hdev->cs_mirror_list);
+ spin_lock_init(&hdev->cs_mirror_lock);
INIT_LIST_HEAD(&hdev->fpriv_list);
mutex_init(&hdev->fpriv_list_lock);
atomic_set(&hdev->in_reset, 0);
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 40b566c4b791..eb968c30adb9 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -1699,8 +1699,8 @@ struct hl_mmu_funcs {
* @eq_wq: work queue of event queue for executing work in process context.
* @kernel_ctx: Kernel driver context structure.
* @kernel_queues: array of hl_hw_queue.
- * @hw_queues_mirror_list: CS mirror list for TDR.
- * @hw_queues_mirror_lock: protects hw_queues_mirror_list.
+ * @cs_mirror_list: CS mirror list for TDR.
+ * @cs_mirror_lock: protects cs_mirror_list.
* @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
* @event_queue: event queue for IRQ from CPU-CP.
* @dma_pool: DMA pool for small allocations.
@@ -1810,8 +1810,8 @@ struct hl_device {
struct workqueue_struct *eq_wq;
struct hl_ctx *kernel_ctx;
struct hl_hw_queue *kernel_queues;
- struct list_head hw_queues_mirror_list;
- spinlock_t hw_queues_mirror_lock;
+ struct list_head cs_mirror_list;
+ spinlock_t cs_mirror_lock;
struct hl_cb_mgr kernel_cb_mgr;
struct hl_eq event_queue;
struct dma_pool *dma_pool;
diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c
index f9550fcf5500..d1d30fb36410 100644
--- a/drivers/misc/habanalabs/common/hw_queue.c
+++ b/drivers/misc/habanalabs/common/hw_queue.c
@@ -578,20 +578,20 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs)
else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)
hdev->asic_funcs->collective_wait_init_cs(cs);
- spin_lock(&hdev->hw_queues_mirror_lock);
- list_add_tail(&cs->mirror_node, &hdev->hw_queues_mirror_list);
+ spin_lock(&hdev->cs_mirror_lock);
+ list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);
/* Queue TDR if the CS is the first entry and if timeout is wanted */
if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
- (list_first_entry(&hdev->hw_queues_mirror_list,
+ (list_first_entry(&hdev->cs_mirror_list,
struct hl_cs, mirror_node) == cs)) {
cs->tdr_active = true;
schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies);
- spin_unlock(&hdev->hw_queues_mirror_lock);
- } else {
- spin_unlock(&hdev->hw_queues_mirror_lock);
+
}
+ spin_unlock(&hdev->cs_mirror_lock);
+
if (!hdev->cs_active_cnt++) {
struct hl_device_idle_busy_ts *ts;
--
2.17.1
^ permalink raw reply related [flat|nested] 3+ messages in thread