From mboxrd@z Thu Jan 1 00:00:00 1970 From: Olivier Matz Subject: [PATCH v5] mempool: reduce rte_mempool structure size Date: Thu, 14 Apr 2016 11:42:36 +0200 Message-ID: <1460626956-12038-1-git-send-email-olivier.matz@6wind.com> References: <1455302184-48822-1-git-send-email-keith.wiles@intel.com> Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable Cc: thomas.monjalon@6wind.com, pmatilai@redhat.com To: dev@dpdk.org, keith.wiles@intel.com Return-path: Received: from proxy.6wind.com (host.76.145.23.62.rev.coltfrance.com [62.23.145.76]) by dpdk.org (Postfix) with ESMTP id 733C638EB for ; Thu, 14 Apr 2016 11:43:10 +0200 (CEST) In-Reply-To: <1455302184-48822-1-git-send-email-keith.wiles@intel.com> List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Keith Wiles The rte_mempool structure is changed, which will cause an ABI change for this structure. Providing backward compat is not reasonable here as this structure is used in multiple defines/inlines. Allow mempool cache support to be dynamic depending on if the mempool being created needs cache support. Saves about 1.5M of memory used by the rte_mempool structure. Allocating small mempools which do not require cache can consume larges amounts of memory if you have a number of these mempools. Change to be effective in release 16.07. Signed-off-by: Keith Wiles Acked-by: Olivier Matz --- Changes in v5: - use RTE_PTR_ADD() instead of cast to (char *) to fix compilation on til= era. Error log was: rte_mempool.c: In function =E2=80=98rte_mempool_xmem_create=E2=80=99: rte_mempool.c:595: error: cast increases required alignment of target t= ype app/test/test_mempool.c | 4 +-- lib/librte_mempool/rte_mempool.c | 55 ++++++++++++++++++----------------= ------ lib/librte_mempool/rte_mempool.h | 29 ++++++++++----------- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/app/test/test_mempool.c b/app/test/test_mempool.c index f0f823b..10e1fa4 100644 --- a/app/test/test_mempool.c +++ b/app/test/test_mempool.c @@ -122,8 +122,8 @@ test_mempool_basic(void) return -1; =20 printf("get private data\n"); - if (rte_mempool_get_priv(mp) !=3D - (char*) mp + MEMPOOL_HEADER_SIZE(mp, mp->pg_num)) + if (rte_mempool_get_priv(mp) !=3D (char *)mp + + MEMPOOL_HEADER_SIZE(mp, mp->pg_num, mp->cache_size)) return -1; =20 printf("get physical address of an object\n"); diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_me= mpool.c index f8781e1..7a0e07e 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -452,12 +452,8 @@ rte_mempool_xmem_create(const char *name, unsigned n= , unsigned elt_size, /* compilation-time checks */ RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) & RTE_CACHE_LINE_MASK) !=3D 0); -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_cache) & RTE_CACHE_LINE_MASK) !=3D 0); - RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, local_cache) & - RTE_CACHE_LINE_MASK) !=3D 0); -#endif #ifdef RTE_LIBRTE_MEMPOOL_DEBUG RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_debug_stats) & RTE_CACHE_LINE_MASK) !=3D 0); @@ -527,9 +523,8 @@ rte_mempool_xmem_create(const char *name, unsigned n,= unsigned elt_size, */ int head =3D sizeof(struct rte_mempool); int new_size =3D (private_data_size + head) % page_size; - if (new_size) { + if (new_size) private_data_size +=3D page_size - new_size; - } } =20 /* try to allocate tailq entry */ @@ -544,7 +539,8 @@ rte_mempool_xmem_create(const char *name, unsigned n,= unsigned elt_size, * store mempool objects. Otherwise reserve a memzone that is large * enough to hold mempool header and metadata plus mempool objects. */ - mempool_size =3D MEMPOOL_HEADER_SIZE(mp, pg_num) + private_data_size; + mempool_size =3D MEMPOOL_HEADER_SIZE(mp, pg_num, cache_size); + mempool_size +=3D private_data_size; mempool_size =3D RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN); if (vaddr =3D=3D NULL) mempool_size +=3D (size_t)objsz.total_size * n; @@ -591,8 +587,15 @@ rte_mempool_xmem_create(const char *name, unsigned n= , unsigned elt_size, mp->cache_flushthresh =3D CALC_CACHE_FLUSHTHRESH(cache_size); mp->private_data_size =3D private_data_size; =20 + /* + * local_cache pointer is set even if cache_size is zero. + * The local_cache points to just past the elt_pa[] array. + */ + mp->local_cache =3D (struct rte_mempool_cache *) + RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, pg_num, 0)); + /* calculate address of the first element for continuous mempool. */ - obj =3D (char *)mp + MEMPOOL_HEADER_SIZE(mp, pg_num) + + obj =3D (char *)mp + MEMPOOL_HEADER_SIZE(mp, pg_num, cache_size) + private_data_size; obj =3D RTE_PTR_ALIGN_CEIL(obj, RTE_MEMPOOL_ALIGN); =20 @@ -606,9 +609,8 @@ rte_mempool_xmem_create(const char *name, unsigned n,= unsigned elt_size, mp->elt_va_start =3D (uintptr_t)obj; mp->elt_pa[0] =3D mp->phys_addr + (mp->elt_va_start - (uintptr_t)mp); - - /* mempool elements in a separate chunk of memory. */ } else { + /* mempool elements in a separate chunk of memory. */ mp->elt_va_start =3D (uintptr_t)vaddr; memcpy(mp->elt_pa, paddr, sizeof (mp->elt_pa[0]) * pg_num); } @@ -643,19 +645,15 @@ unsigned rte_mempool_count(const struct rte_mempool *mp) { unsigned count; + unsigned lcore_id; =20 count =3D rte_ring_count(mp->ring); =20 -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 - { - unsigned lcore_id; - if (mp->cache_size =3D=3D 0) - return count; + if (mp->cache_size =3D=3D 0) + return count; =20 - for (lcore_id =3D 0; lcore_id < RTE_MAX_LCORE; lcore_id++) - count +=3D mp->local_cache[lcore_id].len; - } -#endif + for (lcore_id =3D 0; lcore_id < RTE_MAX_LCORE; lcore_id++) + count +=3D mp->local_cache[lcore_id].len; =20 /* * due to race condition (access to len is not locked), the @@ -670,13 +668,16 @@ rte_mempool_count(const struct rte_mempool *mp) static unsigned rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) { -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 unsigned lcore_id; unsigned count =3D 0; unsigned cache_count; =20 fprintf(f, " cache infos:\n"); fprintf(f, " cache_size=3D%"PRIu32"\n", mp->cache_size); + + if (mp->cache_size =3D=3D 0) + return count; + for (lcore_id =3D 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { cache_count =3D mp->local_cache[lcore_id].len; fprintf(f, " cache_count[%u]=3D%u\n", lcore_id, cache_count); @@ -684,11 +685,6 @@ rte_mempool_dump_cache(FILE *f, const struct rte_mem= pool *mp) } fprintf(f, " total_cache_count=3D%u\n", count); return count; -#else - RTE_SET_USED(mp); - fprintf(f, " cache disabled\n"); - return 0; -#endif } =20 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG @@ -753,13 +749,16 @@ mempool_audit_cookies(const struct rte_mempool *mp) #define mempool_audit_cookies(mp) do {} while(0) #endif =20 -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /* check cookies before and after objects */ static void mempool_audit_cache(const struct rte_mempool *mp) { /* check cache size consistency */ unsigned lcore_id; + + if (mp->cache_size =3D=3D 0) + return; + for (lcore_id =3D 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { if (mp->local_cache[lcore_id].len > mp->cache_flushthresh) { RTE_LOG(CRIT, MEMPOOL, "badness on cache[%u]\n", @@ -768,10 +767,6 @@ mempool_audit_cache(const struct rte_mempool *mp) } } } -#else -#define mempool_audit_cache(mp) do {} while(0) -#endif - =20 /* check the consistency of mempool (size, cookies, ...) */ void diff --git a/lib/librte_mempool/rte_mempool.h b/lib/librte_mempool/rte_me= mpool.h index 9745bf0..8595e77 100644 --- a/lib/librte_mempool/rte_mempool.h +++ b/lib/librte_mempool/rte_mempool.h @@ -95,7 +95,6 @@ struct rte_mempool_debug_stats { } __rte_cache_aligned; #endif =20 -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /** * A structure that stores a per-core object cache. */ @@ -107,7 +106,6 @@ struct rte_mempool_cache { */ void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3]; /**< Cache objects */ } __rte_cache_aligned; -#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */ =20 /** * A structure that stores the size of mempool elements. @@ -194,10 +192,7 @@ struct rte_mempool { =20 unsigned private_data_size; /**< Size of private data. */ =20 -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 - /** Per-lcore local cache. */ - struct rte_mempool_cache local_cache[RTE_MAX_LCORE]; -#endif + struct rte_mempool_cache *local_cache; /**< Per-lcore local cache */ =20 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG /** Per-lcore statistics. */ @@ -247,6 +242,13 @@ struct rte_mempool { #endif =20 /** + * Size of elt_pa array size based on number of pages. (Internal use) + */ +#define __PA_SIZE(mp, pgn) \ + RTE_ALIGN_CEIL((((pgn) - RTE_DIM((mp)->elt_pa)) * \ + sizeof((mp)->elt_pa[0])), RTE_CACHE_LINE_SIZE) + +/** * Calculate the size of the mempool header. * * @param mp @@ -254,9 +256,9 @@ struct rte_mempool { * @param pgn * Number of pages used to store mempool objects. */ -#define MEMPOOL_HEADER_SIZE(mp, pgn) (sizeof(*(mp)) + \ - RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \ - sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE)) +#define MEMPOOL_HEADER_SIZE(mp, pgn, cs) \ + (sizeof(*(mp)) + __PA_SIZE(mp, pgn) + (((cs) =3D=3D 0) ? 0 : \ + (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE))) =20 /** * Return true if the whole mempool is in contiguous memory. @@ -755,19 +757,16 @@ static inline void __attribute__((always_inline)) __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table, unsigned n, int is_mp) { -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 struct rte_mempool_cache *cache; uint32_t index; void **cache_objs; unsigned lcore_id =3D rte_lcore_id(); uint32_t cache_size =3D mp->cache_size; uint32_t flushthresh =3D mp->cache_flushthresh; -#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */ =20 /* increment stat now, adding in mempool always success */ __MEMPOOL_STAT_ADD(mp, put, n); =20 -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /* cache is not enabled or single producer or non-EAL thread */ if (unlikely(cache_size =3D=3D 0 || is_mp =3D=3D 0 || lcore_id >=3D RTE_MAX_LCORE)) @@ -802,7 +801,6 @@ __mempool_put_bulk(struct rte_mempool *mp, void * con= st *obj_table, return; =20 ring_enqueue: -#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */ =20 /* push remaining objects in ring */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG @@ -946,7 +944,6 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj= _table, unsigned n, int is_mc) { int ret; -#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 struct rte_mempool_cache *cache; uint32_t index, len; void **cache_objs; @@ -992,7 +989,6 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj= _table, return 0; =20 ring_dequeue: -#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */ =20 /* get remaining objects from ring */ if (is_mc) @@ -1293,7 +1289,8 @@ void rte_mempool_audit(const struct rte_mempool *mp= ); */ static inline void *rte_mempool_get_priv(struct rte_mempool *mp) { - return (char *)mp + MEMPOOL_HEADER_SIZE(mp, mp->pg_num); + return (char *)mp + + MEMPOOL_HEADER_SIZE(mp, mp->pg_num, mp->cache_size); } =20 /** --=20 2.1.4