* [PATCH 1/2] drm/ttm: don't update global memory count for some special cases
@ 2018-01-12 6:14 Roger He
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
` (2 more replies)
0 siblings, 3 replies; 7+ messages in thread
From: Roger He @ 2018-01-12 6:14 UTC (permalink / raw)
To: dri-devel; +Cc: Roger He, Christian.Koenig
add input parameter for ttm_dma_unpopulate.
when ttm_dma_pool_get_pages or ttm_mem_global_alloc_page fail, don't
call ttm_mem_global_free_page to update global memory count.
Signed-off-by: Roger He <Hongbo.He@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +-
drivers/gpu/drm/radeon/radeon_ttm.c | 2 +-
drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 19 +++++++++++--------
drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 2 +-
include/drm/ttm/ttm_page_alloc.h | 5 +++--
6 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index e4bb435..723ccf1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1046,7 +1046,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
- ttm_dma_unpopulate(>t->ttm, adev->dev);
+ ttm_dma_unpopulate(>t->ttm, adev->dev, true);
return;
}
#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index ce328ed..3f7c30f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1632,7 +1632,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
if (swiotlb_nr_tbl()) {
- ttm_dma_unpopulate((void *)ttm, dev);
+ ttm_dma_unpopulate((void *)ttm, dev, true);
return;
}
#endif
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index a0a839b..449cc65 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -789,7 +789,7 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
- ttm_dma_unpopulate(>t->ttm, rdev->dev);
+ ttm_dma_unpopulate(>t->ttm, rdev->dev, true);
return;
}
#endif
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index c7f01a4..4cda764 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -969,7 +969,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
pool->size, ctx);
if (unlikely(ret != 0)) {
- ttm_dma_unpopulate(ttm_dma, dev);
+ ttm_dma_unpopulate(ttm_dma, dev, false);
return -ENOMEM;
}
@@ -998,14 +998,14 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
while (num_pages) {
ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
if (ret != 0) {
- ttm_dma_unpopulate(ttm_dma, dev);
+ ttm_dma_unpopulate(ttm_dma, dev, false);
return -ENOMEM;
}
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
pool->size, ctx);
if (unlikely(ret != 0)) {
- ttm_dma_unpopulate(ttm_dma, dev);
+ ttm_dma_unpopulate(ttm_dma, dev, false);
return -ENOMEM;
}
@@ -1016,7 +1016,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
ret = ttm_tt_swapin(ttm);
if (unlikely(ret != 0)) {
- ttm_dma_unpopulate(ttm_dma, dev);
+ ttm_dma_unpopulate(ttm_dma, dev, true);
return ret;
}
}
@@ -1027,7 +1027,8 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
EXPORT_SYMBOL_GPL(ttm_dma_populate);
/* Put all pages in pages list to correct pool to wait for reuse */
-void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
+void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
+ bool update_glob_count)
{
struct ttm_tt *ttm = &ttm_dma->ttm;
struct dma_pool *pool;
@@ -1049,7 +1050,8 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
continue;
count++;
- ttm_mem_global_free_page(ttm->glob->mem_glob,
+ if (update_glob_count)
+ ttm_mem_global_free_page(ttm->glob->mem_glob,
d_page->p, pool->size);
ttm_dma_page_put(pool, d_page);
}
@@ -1094,11 +1096,12 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
if (is_cached) {
list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
- ttm_mem_global_free_page(ttm->glob->mem_glob,
+ if (update_glob_count)
+ ttm_mem_global_free_page(ttm->glob->mem_glob,
d_page->p, pool->size);
ttm_dma_page_put(pool, d_page);
}
- } else {
+ } else if (update_glob_count) {
for (i = 0; i < count; i++) {
ttm_mem_global_free_page(ttm->glob->mem_glob,
ttm->pages[i], pool->size);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 22231bc..38df05d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -681,7 +681,7 @@ static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
size_t size =
ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
- ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
+ ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev, true);
ttm_mem_global_free(glob, size);
} else
ttm_pool_unpopulate(ttm);
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 4d9b019..8518cd5 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -92,7 +92,8 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data);
int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
struct ttm_operation_ctx *ctx);
-void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev);
+void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
+ bool update_glob_count);
#else
static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob,
@@ -114,7 +115,7 @@ static inline int ttm_dma_populate(struct ttm_dma_tt *ttm_dma,
return -ENOMEM;
}
static inline void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma,
- struct device *dev)
+ struct device *dev, bool update_glob_count);
{
}
#endif
--
2.7.4
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together
2018-01-12 6:14 [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Roger He
@ 2018-01-12 6:14 ` Roger He
2018-01-12 6:55 ` Andrey Grodzovsky
2018-01-12 7:58 ` Christian König
2018-01-12 6:45 ` [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Chunming Zhou
2018-01-12 7:54 ` Christian König
2 siblings, 2 replies; 7+ messages in thread
From: Roger He @ 2018-01-12 6:14 UTC (permalink / raw)
To: dri-devel; +Cc: Roger He, Christian.Koenig
if ttm_get_pages or ttm_mem_global_alloc_page fail, should not update
global memory count.
Signed-off-by: Roger He <Hongbo.He@amd.com>
---
drivers/gpu/drm/ttm/ttm_page_alloc.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 0eab24e..aebabf1 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -1075,7 +1075,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
ttm->caching_state);
if (unlikely(ret != 0)) {
- ttm_pool_unpopulate(ttm);
+ ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+ ttm->caching_state);
return ret;
}
@@ -1083,7 +1084,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
PAGE_SIZE, ctx);
if (unlikely(ret != 0)) {
- ttm_pool_unpopulate(ttm);
+ ttm_put_pages(ttm->pages, ttm->num_pages,
+ ttm->page_flags, ttm->caching_state);
return -ENOMEM;
}
}
--
2.7.4
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [PATCH 1/2] drm/ttm: don't update global memory count for some special cases
2018-01-12 6:14 [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Roger He
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
@ 2018-01-12 6:45 ` Chunming Zhou
2018-01-12 6:52 ` He, Roger
2018-01-12 7:54 ` Christian König
2 siblings, 1 reply; 7+ messages in thread
From: Chunming Zhou @ 2018-01-12 6:45 UTC (permalink / raw)
To: Roger He, dri-devel; +Cc: Christian.Koenig
Can this fix GTT size issue or not?
Regards,
David Zhou
On 2018年01月12日 14:14, Roger He wrote:
> add input parameter for ttm_dma_unpopulate.
> when ttm_dma_pool_get_pages or ttm_mem_global_alloc_page fail, don't
> call ttm_mem_global_free_page to update global memory count.
>
> Signed-off-by: Roger He <Hongbo.He@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
> drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +-
> drivers/gpu/drm/radeon/radeon_ttm.c | 2 +-
> drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 19 +++++++++++--------
> drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 2 +-
> include/drm/ttm/ttm_page_alloc.h | 5 +++--
> 6 files changed, 18 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index e4bb435..723ccf1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1046,7 +1046,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, adev->dev);
> + ttm_dma_unpopulate(>t->ttm, adev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
> index ce328ed..3f7c30f 100644
> --- a/drivers/gpu/drm/nouveau/nouveau_bo.c
> +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
> @@ -1632,7 +1632,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate((void *)ttm, dev);
> + ttm_dma_unpopulate((void *)ttm, dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
> index a0a839b..449cc65 100644
> --- a/drivers/gpu/drm/radeon/radeon_ttm.c
> +++ b/drivers/gpu/drm/radeon/radeon_ttm.c
> @@ -789,7 +789,7 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, rdev->dev);
> + ttm_dma_unpopulate(>t->ttm, rdev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> index c7f01a4..4cda764 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> @@ -969,7 +969,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -998,14 +998,14 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> while (num_pages) {
> ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
> if (ret != 0) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -1016,7 +1016,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
> ret = ttm_tt_swapin(ttm);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, true);
> return ret;
> }
> }
> @@ -1027,7 +1027,8 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> EXPORT_SYMBOL_GPL(ttm_dma_populate);
>
> /* Put all pages in pages list to correct pool to wait for reuse */
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count)
> {
> struct ttm_tt *ttm = &ttm_dma->ttm;
> struct dma_pool *pool;
> @@ -1049,7 +1050,8 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
> continue;
>
> count++;
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> @@ -1094,11 +1096,12 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
>
> if (is_cached) {
> list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> - } else {
> + } else if (update_glob_count) {
> for (i = 0; i < count; i++) {
> ttm_mem_global_free_page(ttm->glob->mem_glob,
> ttm->pages[i], pool->size);
> diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> index 22231bc..38df05d 100644
> --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> @@ -681,7 +681,7 @@ static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
> size_t size =
> ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
>
> - ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
> + ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev, true);
> ttm_mem_global_free(glob, size);
> } else
> ttm_pool_unpopulate(ttm);
> diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
> index 4d9b019..8518cd5 100644
> --- a/include/drm/ttm/ttm_page_alloc.h
> +++ b/include/drm/ttm/ttm_page_alloc.h
> @@ -92,7 +92,8 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data);
>
> int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> struct ttm_operation_ctx *ctx);
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev);
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count);
>
> #else
> static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob,
> @@ -114,7 +115,7 @@ static inline int ttm_dma_populate(struct ttm_dma_tt *ttm_dma,
> return -ENOMEM;
> }
> static inline void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma,
> - struct device *dev)
> + struct device *dev, bool update_glob_count);
> {
> }
> #endif
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* RE: [PATCH 1/2] drm/ttm: don't update global memory count for some special cases
2018-01-12 6:45 ` [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Chunming Zhou
@ 2018-01-12 6:52 ` He, Roger
0 siblings, 0 replies; 7+ messages in thread
From: He, Roger @ 2018-01-12 6:52 UTC (permalink / raw)
To: Zhou, David(ChunMing), dri-devel@lists.freedesktop.org; +Cc: Koenig, Christian
It is the first step for the GTT size issue.
Need other patches I am working on it.
Thanks
Roger(Hongbo.He)
-----Original Message-----
From: Zhou, David(ChunMing)
Sent: Friday, January 12, 2018 2:46 PM
To: He, Roger <Hongbo.He@amd.com>; dri-devel@lists.freedesktop.org
Cc: Koenig, Christian <Christian.Koenig@amd.com>
Subject: Re: [PATCH 1/2] drm/ttm: don't update global memory count for some special cases
Can this fix GTT size issue or not?
Regards,
David Zhou
On 2018年01月12日 14:14, Roger He wrote:
> add input parameter for ttm_dma_unpopulate.
> when ttm_dma_pool_get_pages or ttm_mem_global_alloc_page fail, don't
> call ttm_mem_global_free_page to update global memory count.
>
> Signed-off-by: Roger He <Hongbo.He@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
> drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +-
> drivers/gpu/drm/radeon/radeon_ttm.c | 2 +-
> drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 19 +++++++++++--------
> drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 2 +-
> include/drm/ttm/ttm_page_alloc.h | 5 +++--
> 6 files changed, 18 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index e4bb435..723ccf1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1046,7 +1046,7 @@ static void amdgpu_ttm_tt_unpopulate(struct
> ttm_tt *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, adev->dev);
> + ttm_dma_unpopulate(>t->ttm, adev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c
> b/drivers/gpu/drm/nouveau/nouveau_bo.c
> index ce328ed..3f7c30f 100644
> --- a/drivers/gpu/drm/nouveau/nouveau_bo.c
> +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
> @@ -1632,7 +1632,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate((void *)ttm, dev);
> + ttm_dma_unpopulate((void *)ttm, dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c
> b/drivers/gpu/drm/radeon/radeon_ttm.c
> index a0a839b..449cc65 100644
> --- a/drivers/gpu/drm/radeon/radeon_ttm.c
> +++ b/drivers/gpu/drm/radeon/radeon_ttm.c
> @@ -789,7 +789,7 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt
> *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, rdev->dev);
> + ttm_dma_unpopulate(>t->ttm, rdev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> index c7f01a4..4cda764 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> @@ -969,7 +969,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -998,14 +998,14 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> while (num_pages) {
> ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
> if (ret != 0) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -1016,7 +1016,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
> ret = ttm_tt_swapin(ttm);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, true);
> return ret;
> }
> }
> @@ -1027,7 +1027,8 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> EXPORT_SYMBOL_GPL(ttm_dma_populate);
>
> /* Put all pages in pages list to correct pool to wait for reuse */
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device
> *dev)
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count)
> {
> struct ttm_tt *ttm = &ttm_dma->ttm;
> struct dma_pool *pool;
> @@ -1049,7 +1050,8 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
> continue;
>
> count++;
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> @@ -1094,11 +1096,12 @@ void ttm_dma_unpopulate(struct ttm_dma_tt
> *ttm_dma, struct device *dev)
>
> if (is_cached) {
> list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> - } else {
> + } else if (update_glob_count) {
> for (i = 0; i < count; i++) {
> ttm_mem_global_free_page(ttm->glob->mem_glob,
> ttm->pages[i], pool->size);
> diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> index 22231bc..38df05d 100644
> --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> @@ -681,7 +681,7 @@ static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
> size_t size =
> ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
>
> - ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
> + ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev, true);
> ttm_mem_global_free(glob, size);
> } else
> ttm_pool_unpopulate(ttm);
> diff --git a/include/drm/ttm/ttm_page_alloc.h
> b/include/drm/ttm/ttm_page_alloc.h
> index 4d9b019..8518cd5 100644
> --- a/include/drm/ttm/ttm_page_alloc.h
> +++ b/include/drm/ttm/ttm_page_alloc.h
> @@ -92,7 +92,8 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m,
> void *data);
>
> int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> struct ttm_operation_ctx *ctx);
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device
> *dev);
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count);
>
> #else
> static inline int ttm_dma_page_alloc_init(struct ttm_mem_global
> *glob, @@ -114,7 +115,7 @@ static inline int ttm_dma_populate(struct ttm_dma_tt *ttm_dma,
> return -ENOMEM;
> }
> static inline void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma,
> - struct device *dev)
> + struct device *dev, bool update_glob_count);
> {
> }
> #endif
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
@ 2018-01-12 6:55 ` Andrey Grodzovsky
2018-01-12 7:58 ` Christian König
1 sibling, 0 replies; 7+ messages in thread
From: Andrey Grodzovsky @ 2018-01-12 6:55 UTC (permalink / raw)
To: Roger He, dri-devel; +Cc: Christian.Koenig
Both patches
Tested-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com>
Acked-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com>
Andrey
On 01/12/2018 01:14 AM, Roger He wrote:
> if ttm_get_pages or ttm_mem_global_alloc_page fail, should not update
> global memory count.
>
> Signed-off-by: Roger He <Hongbo.He@amd.com>
> ---
> drivers/gpu/drm/ttm/ttm_page_alloc.c | 6 ++++--
> 1 file changed, 4 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> index 0eab24e..aebabf1 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> @@ -1075,7 +1075,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
> ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
> ttm->caching_state);
> if (unlikely(ret != 0)) {
> - ttm_pool_unpopulate(ttm);
> + ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
> + ttm->caching_state);
> return ret;
> }
>
> @@ -1083,7 +1084,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> PAGE_SIZE, ctx);
> if (unlikely(ret != 0)) {
> - ttm_pool_unpopulate(ttm);
> + ttm_put_pages(ttm->pages, ttm->num_pages,
> + ttm->page_flags, ttm->caching_state);
> return -ENOMEM;
> }
> }
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 1/2] drm/ttm: don't update global memory count for some special cases
2018-01-12 6:14 [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Roger He
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
2018-01-12 6:45 ` [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Chunming Zhou
@ 2018-01-12 7:54 ` Christian König
2 siblings, 0 replies; 7+ messages in thread
From: Christian König @ 2018-01-12 7:54 UTC (permalink / raw)
To: Roger He, dri-devel; +Cc: Christian.Koenig
Am 12.01.2018 um 07:14 schrieb Roger He:
> add input parameter for ttm_dma_unpopulate.
> when ttm_dma_pool_get_pages or ttm_mem_global_alloc_page fail, don't
> call ttm_mem_global_free_page to update global memory count.
>
> Signed-off-by: Roger He <Hongbo.He@amd.com>
Good catch, but that doesn't looks correct to me.
See the handling in ttm_dma_populate():
> ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
> if (ret != 0)
> break;
>
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> ttm_dma_unpopulate(ttm_dma, dev);
> return -ENOMEM;
> }
>
When ttm_mem_global_alloc_page() fails we only missed to account the
last allocated page, not all of them.
Additional to that I really don't like adding the new parameter to the
drivers.
Regards,
Christian.
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
> drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +-
> drivers/gpu/drm/radeon/radeon_ttm.c | 2 +-
> drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 19 +++++++++++--------
> drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 2 +-
> include/drm/ttm/ttm_page_alloc.h | 5 +++--
> 6 files changed, 18 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index e4bb435..723ccf1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1046,7 +1046,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, adev->dev);
> + ttm_dma_unpopulate(>t->ttm, adev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
> index ce328ed..3f7c30f 100644
> --- a/drivers/gpu/drm/nouveau/nouveau_bo.c
> +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
> @@ -1632,7 +1632,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate((void *)ttm, dev);
> + ttm_dma_unpopulate((void *)ttm, dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
> index a0a839b..449cc65 100644
> --- a/drivers/gpu/drm/radeon/radeon_ttm.c
> +++ b/drivers/gpu/drm/radeon/radeon_ttm.c
> @@ -789,7 +789,7 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
>
> #ifdef CONFIG_SWIOTLB
> if (swiotlb_nr_tbl()) {
> - ttm_dma_unpopulate(>t->ttm, rdev->dev);
> + ttm_dma_unpopulate(>t->ttm, rdev->dev, true);
> return;
> }
> #endif
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> index c7f01a4..4cda764 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
> @@ -969,7 +969,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -998,14 +998,14 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> while (num_pages) {
> ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
> if (ret != 0) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> pool->size, ctx);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, false);
> return -ENOMEM;
> }
>
> @@ -1016,7 +1016,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
> ret = ttm_tt_swapin(ttm);
> if (unlikely(ret != 0)) {
> - ttm_dma_unpopulate(ttm_dma, dev);
> + ttm_dma_unpopulate(ttm_dma, dev, true);
> return ret;
> }
> }
> @@ -1027,7 +1027,8 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> EXPORT_SYMBOL_GPL(ttm_dma_populate);
>
> /* Put all pages in pages list to correct pool to wait for reuse */
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count)
> {
> struct ttm_tt *ttm = &ttm_dma->ttm;
> struct dma_pool *pool;
> @@ -1049,7 +1050,8 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
> continue;
>
> count++;
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> @@ -1094,11 +1096,12 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
>
> if (is_cached) {
> list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
> - ttm_mem_global_free_page(ttm->glob->mem_glob,
> + if (update_glob_count)
> + ttm_mem_global_free_page(ttm->glob->mem_glob,
> d_page->p, pool->size);
> ttm_dma_page_put(pool, d_page);
> }
> - } else {
> + } else if (update_glob_count) {
> for (i = 0; i < count; i++) {
> ttm_mem_global_free_page(ttm->glob->mem_glob,
> ttm->pages[i], pool->size);
> diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> index 22231bc..38df05d 100644
> --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
> @@ -681,7 +681,7 @@ static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
> size_t size =
> ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
>
> - ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
> + ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev, true);
> ttm_mem_global_free(glob, size);
> } else
> ttm_pool_unpopulate(ttm);
> diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
> index 4d9b019..8518cd5 100644
> --- a/include/drm/ttm/ttm_page_alloc.h
> +++ b/include/drm/ttm/ttm_page_alloc.h
> @@ -92,7 +92,8 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data);
>
> int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> struct ttm_operation_ctx *ctx);
> -void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev);
> +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev,
> + bool update_glob_count);
>
> #else
> static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob,
> @@ -114,7 +115,7 @@ static inline int ttm_dma_populate(struct ttm_dma_tt *ttm_dma,
> return -ENOMEM;
> }
> static inline void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma,
> - struct device *dev)
> + struct device *dev, bool update_glob_count);
> {
> }
> #endif
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
2018-01-12 6:55 ` Andrey Grodzovsky
@ 2018-01-12 7:58 ` Christian König
1 sibling, 0 replies; 7+ messages in thread
From: Christian König @ 2018-01-12 7:58 UTC (permalink / raw)
To: Roger He, dri-devel; +Cc: Christian.Koenig
Am 12.01.2018 um 07:14 schrieb Roger He:
> if ttm_get_pages or ttm_mem_global_alloc_page fail, should not update
> global memory count.
>
> Signed-off-by: Roger He <Hongbo.He@amd.com>
> ---
> drivers/gpu/drm/ttm/ttm_page_alloc.c | 6 ++++--
> 1 file changed, 4 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> index 0eab24e..aebabf1 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> @@ -1075,7 +1075,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
> ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
> ttm->caching_state);
> if (unlikely(ret != 0)) {
> - ttm_pool_unpopulate(ttm);
> + ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
> + ttm->caching_state);
That is not correctly indented. The "ttm->caching_state" should be one
the same column as the "ttm->pages" above. It looks like your editor
only inserts tabs, not spaces.
Apart from that the patch is Reviewed-by: Christian König
<christian.koenig@amd.com>.
Regards,
Christian.
> return ret;
> }
>
> @@ -1083,7 +1084,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
> ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
> PAGE_SIZE, ctx);
> if (unlikely(ret != 0)) {
> - ttm_pool_unpopulate(ttm);
> + ttm_put_pages(ttm->pages, ttm->num_pages,
> + ttm->page_flags, ttm->caching_state);
> return -ENOMEM;
> }
> }
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2018-01-12 13:58 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-01-12 6:14 [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Roger He
2018-01-12 6:14 ` [PATCH 2/2] drm/ttm: only free pages rather than update global memory count together Roger He
2018-01-12 6:55 ` Andrey Grodzovsky
2018-01-12 7:58 ` Christian König
2018-01-12 6:45 ` [PATCH 1/2] drm/ttm: don't update global memory count for some special cases Chunming Zhou
2018-01-12 6:52 ` He, Roger
2018-01-12 7:54 ` Christian König
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox