Re: [PATCH] drm/amdgpu: set vm size and block size by individual gmc by default

["Zhang, Jerry (Junwei)" <Jerry.Zhang-5C7GfCeVMHo@public.gmane.org>]

Thanks you all for the comments
Updated the v2, please help review

On 04/05/2017 09:39 PM, Christian König wrote:
> Am 05.04.2017 um 15:32 schrieb Alex Deucher:
>> On Wed, Apr 5, 2017 at 5:01 AM, Christian König <deathsimple@vodafone.de> wrote:
>>> Am 05.04.2017 um 08:43 schrieb Junwei Zhang:
>>>> By default, the value is set by individual gmc.
>>>> if a specific value is input, it overrides the global value for all
>>>>
>>>> Signed-off-by: Junwei Zhang <Jerry.Zhang@amd.com>
>>>> ---
>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  2 ++
>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 44
>>>> ++++++++++++++++++------------
>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    |  2 +-
>>>>    drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c      | 10 +++++--
>>>>    drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c      | 10 +++++--
>>>>    drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c      | 10 +++++--
>>>>    drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c      | 30 ++++++++++++++------
>>>>    7 files changed, 74 insertions(+), 34 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> index 65021df..d7f75ce 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> @@ -1647,6 +1647,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev,
>>>> uint32_t reg, uint32_t v,
>>>>    bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
>>>>    bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
>>>>    +uint32_t amdgpu_get_block_size(int vm_size);
>>>> +
>>>>    /*
>>>>     * Registers read & write functions.
>>>>     */
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> index 1d0c742..2f91c2f 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> @@ -1039,35 +1039,44 @@ static bool amdgpu_check_pot_argument(int arg)
>>>>          return (arg & (arg - 1)) == 0;
>>>>    }
>>>>    -static void amdgpu_get_block_size(struct amdgpu_device *adev)
>>>> +uint32_t amdgpu_get_block_size(int vm_size)
>>>> +{
>>>> +       /* Total bits covered by PD + PTs */
>>>> +       unsigned bits = ilog2(vm_size) + 18;
>>>> +
>>>> +       /* Make sure the PD is 4K in size up to 8GB address space.
>>>> +          Above that split equal between PD and PTs */
>>>> +       if (vm_size <= 8)
>>>> +               return (bits - 9);
>>>> +       else
>>>> +               return ((bits + 3) / 2);
>>>> +}
>>>
>>> Please move that helper into amdgpu_vmc.c.
>>>
>>>
>>>> +
>>>> +static void amdgpu_check_block_size(struct amdgpu_device *adev)
>>>>    {
>>>>          /* defines number of bits in page table versus page directory,
>>>>           * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
>>>>           * page table and the remaining bits are in the page directory */
>>>> -       if (amdgpu_vm_block_size == -1) {
>>>> -
>>>> -               /* Total bits covered by PD + PTs */
>>>> -               unsigned bits = ilog2(amdgpu_vm_size) + 18;
>>>> -
>>>> -               /* Make sure the PD is 4K in size up to 8GB address space.
>>>> -                  Above that split equal between PD and PTs */
>>>> -               if (amdgpu_vm_size <= 8)
>>>> -                       amdgpu_vm_block_size = bits - 9;
>>>> -               else
>>>> -                       amdgpu_vm_block_size = (bits + 3) / 2;
>>>> +       if (amdgpu_vm_block_size == -1)
>>>> +               return;
>>>>    -     } else if (amdgpu_vm_block_size < 9) {
>>>> +       if (amdgpu_vm_block_size < 9) {
>>>>                  dev_warn(adev->dev, "VM page table size (%d) too small\n",
>>>>                           amdgpu_vm_block_size);
>>>> -               amdgpu_vm_block_size = 9;
>>>> +               goto def_value;
>>>>          }
>>>>          if (amdgpu_vm_block_size > 24 ||
>>>>              (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
>>>>                  dev_warn(adev->dev, "VM page table size (%d) too large\n",
>>>>                           amdgpu_vm_block_size);
>>>> -               amdgpu_vm_block_size = 9;
>>>> +               goto def_value;
>>>>          }
>>>> +
>>>> +       return;
>>>> +
>>>> +def_value:
>>>> +       amdgpu_vm_block_size = -1;
>>>>    }
>>>>      static void amdgpu_check_vm_size(struct amdgpu_device *adev)
>>>> @@ -1096,8 +1105,7 @@ static void amdgpu_check_vm_size(struct
>>>> amdgpu_device *adev)
>>>>          return;
>>>>      def_value:
>>>> -       amdgpu_vm_size = 8;
>>>> -       dev_info(adev->dev, "set default VM size %dGB\n", amdgpu_vm_size);
>>>> +       amdgpu_vm_size = -1;;
>>>>    }
>>>>      /**
>>>> @@ -1131,7 +1139,7 @@ static void amdgpu_check_arguments(struct
>>>> amdgpu_device *adev)
>>>>          amdgpu_check_vm_size(adev);
>>>>    -     amdgpu_get_block_size(adev);
>>>> +       amdgpu_check_block_size(adev);
>>>>          if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16
>>>> ||
>>>>              !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> index bfd945b..6238e2e 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> @@ -86,7 +86,7 @@
>>>>    unsigned amdgpu_ip_block_mask = 0xffffffff;
>>>>    int amdgpu_bapm = -1;
>>>>    int amdgpu_deep_color = 0;
>>>> -int amdgpu_vm_size = 64;
>>>> +int amdgpu_vm_size = -1;
>>>>    int amdgpu_vm_block_size = -1;
>>>>    int amdgpu_vm_fault_stop = 0;
>>>>    int amdgpu_vm_debug = 0;
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
>>>> index 30d5c42..f46c52c 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
>>>> @@ -849,8 +849,14 @@ static int gmc_v6_0_sw_init(void *handle)
>>>>          if (r)
>>>>                  return r;
>>>>    -     adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> -       adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       if (amdgpu_vm_size == -1) {
>>>> +               adev->vm_manager.vm_size = 64;
>>>> +               adev->vm_manager.block_size =
>>>> +                       amdgpu_get_block_size(adev->vm_manager.vm_size);
>>>> +       } else {
>>>> +               adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> +               adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       }
>>>>          adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
>>>>          DRM_INFO("vm size is %d GB, block size is %d-bit\n",
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
>>>> index 7113765..80b31f4 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
>>>> @@ -1003,8 +1003,14 @@ static int gmc_v7_0_sw_init(void *handle)
>>>>           * Currently set to 4GB ((1 << 20) 4k pages).
>>>>           * Max GPUVM size for cayman and SI is 40 bits.
>>>>           */
>>>> -       adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> -       adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       if (amdgpu_vm_size == -1) {
>>>> +               adev->vm_manager.vm_size = 64;
>>>> +               adev->vm_manager.block_size =
>>>> +                       amdgpu_get_block_size(adev->vm_manager.vm_size);
>>>> +       } else {
>>>> +               adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> +               adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       }
>>>
>>> That's a bit to simple. The logic should probably rather be:
>>>
>>> if (amdgpu_vm_size == -1)
>>>      adev->vm_manager.vm_size = 64;
>>> else
>>>      adev->vm_manager.vm_size = amdgpu_vm_size;
>>>
>>> if (amdgpu_vm_block_size == -1)
>>>      adev->vm_manager.block_size =
>>> amdgpu_get_block_size(adev->vm_manager.vm_size);
>>> else
>>>      adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>
>>>
>>> Since the logic is the same for GFX6,7 and 8 we might put that into a helper
>>> as well.
>>>
>>>
>>>>          adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
>>>>          DRM_INFO("vm size is %d GB, block size is %d-bit\n",
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
>>>> index b3d1f1b..f1f135a 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
>>>> @@ -1087,8 +1087,14 @@ static int gmc_v8_0_sw_init(void *handle)
>>>>           * Currently set to 4GB ((1 << 20) 4k pages).
>>>>           * Max GPUVM size for cayman and SI is 40 bits.
>>>>           */
>>>> -       adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> -       adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       if (amdgpu_vm_size == -1) {
>>>> +               adev->vm_manager.vm_size = 64;
>>>> +               adev->vm_manager.block_size =
>>>> +                       amdgpu_get_block_size(adev->vm_manager.vm_size);
>>>> +       } else {
>>>> +               adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> +               adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       }
>>>>          adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
>>>>          DRM_INFO("vm size is %d GB, block size is %d-bit\n",
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> index 0cae7f0..8e12ff3 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> @@ -532,18 +532,9 @@ static int gmc_v9_0_sw_init(void *handle)
>>>>          if (adev->flags & AMD_IS_APU) {
>>>>                  adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
>>>> -               adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> -               adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>>          } else {
>>>>                  /* XXX Don't know how to get VRAM type yet. */
>>>>                  adev->mc.vram_type = AMDGPU_VRAM_TYPE_HBM;
>>>> -               /*
>>>> -                * To fulfill 4-level page support,
>>>> -                * vm size is 256TB (48bit), maximum size of Vega10,
>>>> -                * block size 512 (9bit)
>>>> -                */
>>>> -               adev->vm_manager.vm_size = 1U << 18;
>>>> -               adev->vm_manager.block_size = 9;
>>>>          }
>>>>          DRM_INFO("vm size is %d GB, block size is %d-bit\n",
>>>> @@ -558,6 +549,27 @@ static int gmc_v9_0_sw_init(void *handle)
>>>>          if (r)
>>>>                  return r;
>>>>    +     if (amdgpu_vm_size == -1) {
>>>> +               if (adev->flags & AMD_IS_APU) {
>>>> +                       adev->vm_manager.vm_size = 64;
>>>> +                       adev->vm_manager.block_size =
>>>> +
>>>> amdgpu_get_block_size(adev->vm_manager.vm_size);
>>>> +               } else {
>>>> +                       /*
>>>> +                        * To fulfill 4-level page support,
>>>> +                        * vm size is 256TB (48bit), maximum size of
>>>> Vega10,
>>>> +                        * block size 512 (9bit)
>>>> +                        */
>>>> +                       adev->vm_manager.vm_size = 1U << 18;
>>>> +                       adev->vm_manager.block_size = 9;
>>>> +               }
>>>> +       } else {
>>>> +               adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> +               adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>> +       }
>>>> +
>>>> +       adev->vm_manager.vm_size = amdgpu_vm_size;
>>>> +       adev->vm_manager.block_size = amdgpu_vm_block_size;
>>>
>>> NAK. On GFX9 vm_size and block_size should be hard coded to 256TB and 9.
>> I think there is still value in being able to override for testing/debugging.
>
> The problem is that this won't work. When you want to override the size you
> need to work with 1 level page tables again.
>
> We could of course try to determine how many levels we want from the VM
> size/block size, but I don't really want to test all those options all the time.
>
> Christian.
>
>>
>> Alex
>>
>>> Christian.
>>>
>>>>          adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
>>>>          /* Set the internal MC address mask
>>>
>>>
>>> _______________________________________________
>>> amd-gfx mailing list
>>> amd-gfx@lists.freedesktop.org
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx