Re: [PATCH 3/3] drm/amdgpu: replace get_user_pages with HMM address mirror helpers v6

Koenig, Christian
2019-02-05T18:09:36Z

Am 05.02.19 um 18:25 schrieb Yang, Philip:
> [SNIP]+
>>> +    if (r == -ERESTARTSYS) {
>>> +        if (!--tries) {
>>> +            DRM_ERROR("Possible deadlock? Retry too many times\n");
>>> +            return -EDEADLK;
>>> +        }
>>> +        goto restart;
>> You really need to restart the IOCTL to potentially handle signals.
>>
>> But since the calling code correctly handles this already you can just
>> drop this change as far as I can see.
>>
> I agree that we should return -ERESTARTSYS to upper layer to handle signals.
>
> But I do not find upper layers handle -ERESTARTSYS in the entire calling
> path, ksys_ioctl -> do_vfs_ioctl -> amdgpu_drm_ioctl -> drm->ioctl ->
> drm_ioctl_kernel -> amdgpu_cs_ioctl. The error code returns to
> application. I confirm it, libdrm userptr test application calling
> amdgpu_cs_ioctl return code is -512, which is -ERESTARTSYS.
>
> So application should handle -ERESTARTSYS to restart the ioctl, but
> libdrm userptr test application doesn't handle this. This causes the
> test failed.

This is a bug in the test cases then.

-ERESTARTSYS can happen at any time during interruptible waiting and it
is mandatory for the upper layer to handle it correctly.

>
> Below are details of userptr path difference. For the previous path,
> libdrm test always goes to step 2, step 3 never trigger. So it never
> return -ERESTARTSYS, but in theory, this could happen.
>
> For HMM path, the test always goes to step 3, we have to handle this
> case inside amdgpu_cs_ioctl. Maybe I can change amdgpu_cs_submit to
> return -EBUSY, then restart the ioctl inside amdgpu_cs_ioctl. I will
> submit new patch.

Clearly a NAK, this won't work correctly.

Christian.

>
> The previous userptr path:
> 1. gem_userptr_ioctl to register userptr
> 2. amdgpu_cs_parser_bos, check if userptr is invalidated, then update
> userptr
> 3. amdgpu_cs_submit, hold p->mn lock, check if userptr is invalidated,
> return -ERESTARTSYS
>
> The new HMM userptr path:
> 1. gem_userptr_ioctl to register userptr
> 2. amdgpu_cs_parser_bos, start HMM to track userptr update
> 3. amdgpu_cs_submit, hold p->mn lock, check HMM if userptr is
> invalidated, return -ERESTARTSYS
>
>
>>> +    }
>>> +
>>>       return r;
>>>   }
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> index d21dd2f369da..555285e329ed 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> @@ -329,26 +329,24 @@ int amdgpu_gem_userptr_ioctl(struct drm_device
>>> *dev, void *data,
>>>           r = amdgpu_bo_reserve(bo, true);
>>>           if (r)
>>> -            goto free_pages;
>>> +            goto user_pages_done;
>>>           amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
>>>           r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
>>>           amdgpu_bo_unreserve(bo);
>>>           if (r)
>>> -            goto free_pages;
>>> +            goto user_pages_done;
>>>       }
>>>       r = drm_gem_handle_create(filp, gobj, &handle);
>>> -    /* drop reference from allocate - handle holds it now */
>>> -    drm_gem_object_put_unlocked(gobj);
>>>       if (r)
>>> -        return r;
>>> +        goto user_pages_done;
>>>       args->handle = handle;
>>> -    return 0;
>>> -free_pages:
>>> -    release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
>>> +user_pages_done:
>>> +    if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
>>> +        amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
>>>   release_object:
>>>       drm_gem_object_put_unlocked(gobj);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
>>> index e356867d2308..fa2516016c43 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
>>> @@ -222,8 +222,6 @@ static void amdgpu_mn_invalidate_node(struct
>>> amdgpu_mn_node *node,
>>>               true, false, MAX_SCHEDULE_TIMEOUT);
>>>           if (r <= 0)
>>>               DRM_ERROR("(%ld) failed to wait for user bo\n", r);
>>> -
>>> -        amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
>>>       }
>>>   }
>>> @@ -504,3 +502,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
>>>       mutex_unlock(&adev->mn_lock);
>>>   }
>>> +/* flags used by HMM internal, not related to CPU/GPU PTE flags */
>>> +static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
>>> +        (1 << 0), /* HMM_PFN_VALID */
>>> +        (1 << 1), /* HMM_PFN_WRITE */
>>> +        0 /* HMM_PFN_DEVICE_PRIVATE */
>>> +};
>>> +
>>> +static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
>>> +        0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
>>> +        0, /* HMM_PFN_NONE */
>>> +        0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
>>> +};
>>> +
>>> +void amdgpu_hmm_init_range(struct hmm_range *range)
>>> +{
>>> +    if (range) {
>>> +        range->flags = hmm_range_flags;
>>> +        range->values = hmm_range_values;
>>> +        range->pfn_shift = PAGE_SHIFT;
>>> +        range->pfns = NULL;
>>> +        INIT_LIST_HEAD(&range->list);
>>> +    }
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
>>> index 0a51fd00021c..4803e216e174 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
>>> @@ -25,9 +25,10 @@
>>>   #define __AMDGPU_MN_H__
>>>   /*
>>> - * MMU Notifier
>>> + * HMM mirror
>>>    */
>>>   struct amdgpu_mn;
>>> +struct hmm_range;
>>>   enum amdgpu_mn_type {
>>>       AMDGPU_MN_TYPE_GFX,
>>> @@ -41,6 +42,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device
>>> *adev,
>>>                   enum amdgpu_mn_type type);
>>>   int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
>>>   void amdgpu_mn_unregister(struct amdgpu_bo *bo);
>>> +void amdgpu_hmm_init_range(struct hmm_range *range);
>>>   #else
>>>   static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
>>>   static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 73e71e61dc99..3fd7851da0c0 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -43,6 +43,7 @@
>>>   #include <linux/pagemap.h>
>>>   #include <linux/debugfs.h>
>>>   #include <linux/iommu.h>
>>> +#include <linux/hmm.h>
>>>   #include "amdgpu.h"
>>>   #include "amdgpu_object.h"
>>>   #include "amdgpu_trace.h"
>>> @@ -705,11 +706,6 @@ static unsigned long amdgpu_ttm_io_mem_pfn(struct
>>> ttm_buffer_object *bo,
>>>   /*
>>>    * TTM backend functions.
>>>    */
>>> -struct amdgpu_ttm_gup_task_list {
>>> -    struct list_head    list;
>>> -    struct task_struct    *task;
>>> -};
>>> -
>>>   struct amdgpu_ttm_tt {
>>>       struct ttm_dma_tt    ttm;
>>>       u64            offset;
>>> @@ -718,85 +714,96 @@ struct amdgpu_ttm_tt {
>>>       uint32_t        userflags;
>>>       spinlock_t              guptasklock;
>>>       struct list_head        guptasks;
>> Those fields are now also completely superfluous.
>>
>> Apart from those two comments that now looks really good to me.
>>
>> Regards,
>> Christian.
>>
> Yes, those fields are superfluous, I remove it in new patch. Thanks.
>
>>> -    atomic_t        mmu_invalidations;
>>> -    uint32_t        last_set_pages;
>>> +    struct hmm_range    range;
>>>   };
>>>   /**
>>> - * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a
>>> USERPTR
>>> - * pointer to memory
>>> + * amdgpu_ttm_tt_get_user_pages - get device accessible pages that
>>> back user
>>> + * memory and start HMM tracking CPU page table update
>>>    *
>>> - * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
>>> - * This provides a wrapper around the get_user_pages() call to provide
>>> - * device accessible pages that back user memory.
>>> + * Calling function must call amdgpu_ttm_tt_userptr_range_done() once
>>> and only
>>> + * once afterwards to stop HMM tracking
>>>    */
>>>   int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page
>>> **pages)
>>>   {
>>>       struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>>       struct mm_struct *mm = gtt->usertask->mm;
>>> -    unsigned int flags = 0;
>>> -    unsigned pinned = 0;
>>> -    int r;
>>> +    unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
>>> +    struct hmm_range *range = &gtt->range;
>>> +    int r = 0, i;
>>>       if (!mm) /* Happens during process shutdown */
>>>           return -ESRCH;
>>> -    if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
>>> -        flags |= FOLL_WRITE;
>>> +    amdgpu_hmm_init_range(range);
>>>       down_read(&mm->mmap_sem);
>>> -    if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
>>> -        /*
>>> -         * check that we only use anonymous memory to prevent problems
>>> -         * with writeback
>>> -         */
>>> -        unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
>>> -        struct vm_area_struct *vma;
>>> +    range->vma = find_vma(mm, gtt->userptr);
>>> +    if (!range_in_vma(range->vma, gtt->userptr, end))
>>> +        r = -EFAULT;
>>> +    else if ((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
>>> +        range->vma->vm_file)
>>> +        r = -EPERM;
>>> +    if (r)
>>> +        goto out;
>>> -        vma = find_vma(mm, gtt->userptr);
>>> -        if (!vma || vma->vm_file || vma->vm_end < end) {
>>> -            up_read(&mm->mmap_sem);
>>> -            return -EPERM;
>>> -        }
>>> +    range->pfns = kvmalloc_array(ttm->num_pages, sizeof(uint64_t),
>>> +                     GFP_KERNEL);
>>> +    if (range->pfns == NULL) {
>>> +        r = -ENOMEM;
>>> +        goto out;
>>>       }
>>> +    range->start = gtt->userptr;
>>> +    range->end = end;
>>> -    /* loop enough times using contiguous pages of memory */
>>> -    do {
>>> -        unsigned num_pages = ttm->num_pages - pinned;
>>> -        uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
>>> -        struct page **p = pages + pinned;
>>> -        struct amdgpu_ttm_gup_task_list guptask;
>>> +    range->pfns[0] = range->flags[HMM_PFN_VALID];
>>> +    range->pfns[0] |= amdgpu_ttm_tt_is_readonly(ttm) ?
>>> +                0 : range->flags[HMM_PFN_WRITE];
>>> +    for (i = 1; i < ttm->num_pages; i++)
>>> +        range->pfns[i] = range->pfns[0];
>>> -        guptask.task = current;
>>> -        spin_lock(&gtt->guptasklock);
>>> -        list_add(&guptask.list, &gtt->guptasks);
>>> -        spin_unlock(&gtt->guptasklock);
>>> +    /* This may trigger page table update */
>>> +    r = hmm_vma_fault(range, true);
>>> +    if (r)
>>> +        goto out_free_pfns;
>>> -        if (mm == current->mm)
>>> -            r = get_user_pages(userptr, num_pages, flags, p, NULL);
>>> -        else
>>> -            r = get_user_pages_remote(gtt->usertask,
>>> -                    mm, userptr, num_pages,
>>> -                    flags, p, NULL, NULL);
>>> +    up_read(&mm->mmap_sem);
>>> -        spin_lock(&gtt->guptasklock);
>>> -        list_del(&guptask.list);
>>> -        spin_unlock(&gtt->guptasklock);
>>> +    for (i = 0; i < ttm->num_pages; i++)
>>> +        pages[i] = hmm_pfn_to_page(range, range->pfns[i]);
>>> -        if (r < 0)
>>> -            goto release_pages;
>>> +    return 0;
>>> -        pinned += r;
>>> +out_free_pfns:
>>> +    kvfree(range->pfns);
>>> +    range->pfns = NULL;
>>> +out:
>>> +    up_read(&mm->mmap_sem);
>>> +    return r;
>>> +}
>>> -    } while (pinned < ttm->num_pages);
>>> +/**
>>> + * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page
>>> table change
>>> + * Check if the pages backing this ttm range have been invalidated
>>> + *
>>> + * Returns: true if pages are still valid
>>> + */
>>> +bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
>>> +{
>>> +    struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>> +    bool r = false;
>>> -    up_read(&mm->mmap_sem);
>>> -    return 0;
>>> +    if (!gtt || !gtt->userptr)
>>> +        return false;
>>> +
>>> +    WARN_ONCE(!gtt->range.pfns, "No user pages to check\n");
>>> +    if (gtt->range.pfns) {
>>> +        r = hmm_vma_range_done(&gtt->range);
>>> +        kvfree(gtt->range.pfns);
>>> +        gtt->range.pfns = NULL;
>>> +    }
>>> -release_pages:
>>> -    release_pages(pages, pinned);
>>> -    up_read(&mm->mmap_sem);
>>>       return r;
>>>   }
>>> @@ -809,16 +816,10 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt
>>> *ttm, struct page **pages)
>>>    */
>>>   void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page
>>> **pages)
>>>   {
>>> -    struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>>       unsigned i;
>>> -    gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
>>> -    for (i = 0; i < ttm->num_pages; ++i) {
>>> -        if (ttm->pages[i])
>>> -            put_page(ttm->pages[i]);
>>> -
>>> +    for (i = 0; i < ttm->num_pages; ++i)
>>>           ttm->pages[i] = pages ? pages[i] : NULL;
>>> -    }
>>>   }
>>>   /**
>>> @@ -903,10 +904,11 @@ static void amdgpu_ttm_tt_unpin_userptr(struct
>>> ttm_tt *ttm)
>>>       /* unmap the pages mapped to the device */
>>>       dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
>>> -    /* mark the pages as dirty */
>>> -    amdgpu_ttm_tt_mark_user_pages(ttm);
>>> -
>>>       sg_free_table(ttm->sg);
>>> +
>>> +    if (gtt->range.pfns &&
>>> +        ttm->pages[0] == hmm_pfn_to_page(&gtt->range,
>>> gtt->range.pfns[0]))
>>> +        WARN_ONCE(1, "Missing get_user_page_done\n");
>>>   }
>>>   int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
>>> @@ -1258,8 +1260,6 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt
>>> *ttm, uint64_t addr,
>>>       spin_lock_init(&gtt->guptasklock);
>>>       INIT_LIST_HEAD(&gtt->guptasks);
>>> -    atomic_set(&gtt->mmu_invalidations, 0);
>>> -    gtt->last_set_pages = 0;
>>>       return 0;
>>>   }
>>> @@ -1289,7 +1289,6 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt
>>> *ttm, unsigned long start,
>>>                     unsigned long end)
>>>   {
>>>       struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>> -    struct amdgpu_ttm_gup_task_list *entry;
>>>       unsigned long size;
>>>       if (gtt == NULL || !gtt->userptr)
>>> @@ -1302,48 +1301,20 @@ bool amdgpu_ttm_tt_affect_userptr(struct
>>> ttm_tt *ttm, unsigned long start,
>>>       if (gtt->userptr > end || gtt->userptr + size <= start)
>>>           return false;
>>> -    /* Search the lists of tasks that hold this mapping and see
>>> -     * if current is one of them.  If it is return false.
>>> -     */
>>> -    spin_lock(&gtt->guptasklock);
>>> -    list_for_each_entry(entry, &gtt->guptasks, list) {
>>> -        if (entry->task == current) {
>>> -            spin_unlock(&gtt->guptasklock);
>>> -            return false;
>>> -        }
>>> -    }
>>> -    spin_unlock(&gtt->guptasklock);
>>> -
>>> -    atomic_inc(&gtt->mmu_invalidations);
>>> -
>>>       return true;
>>>   }
>>>   /**
>>> - * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been
>>> invalidated?
>>> - */
>>> -bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
>>> -                       int *last_invalidated)
>>> -{
>>> -    struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>> -    int prev_invalidated = *last_invalidated;
>>> -
>>> -    *last_invalidated = atomic_read(&gtt->mmu_invalidations);
>>> -    return prev_invalidated != *last_invalidated;
>>> -}
>>> -
>>> -/**
>>> - * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this
>>> ttm_tt object
>>> - * been invalidated since the last time they've been set?
>>> + * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
>>>    */
>>> -bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
>>> +bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
>>>   {
>>>       struct amdgpu_ttm_tt *gtt = (void *)ttm;
>>>       if (gtt == NULL || !gtt->userptr)
>>>           return false;
>>> -    return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
>>> +    return true;
>>>   }
>>>   /**
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index b5b2d101f7db..8988c87fff9d 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -102,6 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object
>>> *bo);
>>>   int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
>>>   int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page
>>> **pages);
>>> +bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
>>>   void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page
>>> **pages);
>>>   void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
>>>   int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
>>> @@ -112,7 +113,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt
>>> *ttm, unsigned long start,
>>>                     unsigned long end);
>>>   bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
>>>                          int *last_invalidated);
>>> -bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
>>> +bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
>>>   bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
>>>   uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct
>>> ttm_mem_reg *mem);
>>>   uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct
>>> ttm_tt *ttm,