DPDK 21.11.1
rte_mempool.h
Go to the documentation of this file.
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright(c) 2016 6WIND S.A.
4 */
5
6#ifndef _RTE_MEMPOOL_H_
7#define _RTE_MEMPOOL_H_
8
36#include <stdio.h>
37#include <stdlib.h>
38#include <stdint.h>
39#include <errno.h>
40#include <inttypes.h>
41
42#include <rte_config.h>
43#include <rte_spinlock.h>
44#include <rte_log.h>
45#include <rte_debug.h>
46#include <rte_lcore.h>
47#include <rte_memory.h>
49#include <rte_ring.h>
50#include <rte_memcpy.h>
51#include <rte_common.h>
52
54
55#ifdef __cplusplus
56extern "C" {
57#endif
58
59#define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
60#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
61#define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
63#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
70struct rte_mempool_debug_stats {
71 uint64_t put_bulk;
72 uint64_t put_objs;
73 uint64_t put_common_pool_bulk;
74 uint64_t put_common_pool_objs;
75 uint64_t get_common_pool_bulk;
76 uint64_t get_common_pool_objs;
77 uint64_t get_success_bulk;
78 uint64_t get_success_objs;
79 uint64_t get_fail_bulk;
80 uint64_t get_fail_objs;
81 uint64_t get_success_blks;
82 uint64_t get_fail_blks;
84#endif
85
90 uint32_t size;
91 uint32_t flushthresh;
92 uint32_t len;
93 /*
94 * Cache is allocated to this size to allow it to overflow in certain
95 * cases to avoid needless emptying of cache.
96 */
97 void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
99
104 uint32_t elt_size;
105 uint32_t header_size;
106 uint32_t trailer_size;
107 uint32_t total_size;
109};
110
112#define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
113 sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
114#define RTE_MEMPOOL_MZ_PREFIX "MP_"
115
116/* "MP_<name>" */
117#define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
118
119#define MEMPOOL_PG_SHIFT_MAX \
120 RTE_DEPRECATED(MEMPOOL_PG_SHIFT_MAX) (sizeof(uintptr_t) * CHAR_BIT - 1)
121
123#define MEMPOOL_PG_NUM_DEFAULT RTE_DEPRECATED(MEMPOOL_PG_NUM_DEFAULT) 1
124
125#ifndef RTE_MEMPOOL_ALIGN
129#define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
130#endif
131
132#define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
133
145 struct rte_mempool *mp;
147#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
148 uint64_t cookie;
149#endif
150};
151
155RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
156
157#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
158
165struct rte_mempool_objtlr {
166 uint64_t cookie;
167};
168
169#endif
170
174RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
175
180 void *opaque);
181
190 struct rte_mempool *mp;
191 void *addr;
193 size_t len;
195 void *opaque;
196};
197
206 unsigned int contig_block_size;
208
213 char name[RTE_MEMPOOL_NAMESIZE];
215 union {
216 void *pool_data;
217 uint64_t pool_id;
218 };
220 const struct rte_memzone *mz;
221 unsigned int flags;
223 uint32_t size;
224 uint32_t cache_size;
227 uint32_t elt_size;
228 uint32_t header_size;
229 uint32_t trailer_size;
239 int32_t ops_index;
240
243 uint32_t populated_size;
244 struct rte_mempool_objhdr_list elt_list;
245 uint32_t nb_mem_chunks;
246 struct rte_mempool_memhdr_list mem_list;
248#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
250 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
251#endif
253
255#define RTE_MEMPOOL_F_NO_SPREAD 0x0001
260#define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
262#define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
267#define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
269#define RTE_MEMPOOL_F_SP_PUT 0x0004
274#define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
276#define RTE_MEMPOOL_F_SC_GET 0x0008
281#define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
283#define RTE_MEMPOOL_F_POOL_CREATED 0x0010
285#define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
290#define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
292#define RTE_MEMPOOL_F_NON_IO 0x0040
293
297#define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
298 | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
299 | RTE_MEMPOOL_F_SP_PUT \
300 | RTE_MEMPOOL_F_SC_GET \
301 | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
302 )
313#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
314#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
315 unsigned __lcore_id = rte_lcore_id(); \
316 if (__lcore_id < RTE_MAX_LCORE) { \
317 mp->stats[__lcore_id].name += n; \
318 } \
319 } while (0)
320#else
321#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
322#endif
323
332#define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
333 (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
334 (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
335
337#define MEMPOOL_HEADER_SIZE(mp, cs) \
338 RTE_DEPRECATED(MEMPOOL_HEADER_SIZE) RTE_MEMPOOL_HEADER_SIZE(mp, cs)
339
340/* return the header of a mempool object (internal) */
341static inline struct rte_mempool_objhdr *
342rte_mempool_get_header(void *obj)
343{
344 return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
345 sizeof(struct rte_mempool_objhdr));
346}
347
357static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
358{
359 struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
360 return hdr->mp;
361}
362
363/* return the trailer of a mempool object (internal) */
364static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
365{
366 struct rte_mempool *mp = rte_mempool_from_obj(obj);
367 return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
368}
369
384void rte_mempool_check_cookies(const struct rte_mempool *mp,
385 void * const *obj_table_const, unsigned n, int free);
386
387#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
388#define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
389 rte_mempool_check_cookies(mp, obj_table_const, n, free)
390#else
391#define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
392#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
393
409void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
410 void * const *first_obj_table_const, unsigned int n, int free);
411
412#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
413#define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
414 free) \
415 rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
416 free)
417#else
418#define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
419 free) \
420 do {} while (0)
421#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
422
423#define RTE_MEMPOOL_OPS_NAMESIZE 32
435typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
436
440typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
441
445typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
446 void * const *obj_table, unsigned int n);
447
451typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
452 void **obj_table, unsigned int n);
453
458 void **first_obj_table, unsigned int n);
459
463typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
464
488typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
489 uint32_t obj_num, uint32_t pg_shift,
490 size_t *min_chunk_size, size_t *align);
491
527ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
528 uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
529 size_t *min_chunk_size, size_t *align);
530
539 uint32_t obj_num, uint32_t pg_shift,
540 size_t *min_chunk_size, size_t *align);
541
555 void *opaque, void *vaddr, rte_iova_t iova);
556
585typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
586 unsigned int max_objs,
587 void *vaddr, rte_iova_t iova, size_t len,
588 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
589
593#define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
594
627int rte_mempool_op_populate_helper(struct rte_mempool *mp,
628 unsigned int flags, unsigned int max_objs,
629 void *vaddr, rte_iova_t iova, size_t len,
630 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
631
639 unsigned int max_objs,
640 void *vaddr, rte_iova_t iova, size_t len,
641 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
642
646typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
647 struct rte_mempool_info *info);
648
649
677
678#define RTE_MEMPOOL_MAX_OPS_IDX 16
691 uint32_t num_ops;
697
700
710static inline struct rte_mempool_ops *
711rte_mempool_get_ops(int ops_index)
712{
713 RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
714
715 return &rte_mempool_ops_table.ops[ops_index];
716}
717
727int
728rte_mempool_ops_alloc(struct rte_mempool *mp);
729
743static inline int
744rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
745 void **obj_table, unsigned n)
746{
747 struct rte_mempool_ops *ops;
748 int ret;
749
750 rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
751 ops = rte_mempool_get_ops(mp->ops_index);
752 ret = ops->dequeue(mp, obj_table, n);
753 if (ret == 0) {
754 RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
755 RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
756 }
757 return ret;
758}
759
773static inline int
774rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
775 void **first_obj_table, unsigned int n)
776{
777 struct rte_mempool_ops *ops;
778
779 ops = rte_mempool_get_ops(mp->ops_index);
780 RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
781 rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
782 return ops->dequeue_contig_blocks(mp, first_obj_table, n);
783}
784
798static inline int
799rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
800 unsigned n)
801{
802 struct rte_mempool_ops *ops;
803
804 RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
805 RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
806 rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
807 ops = rte_mempool_get_ops(mp->ops_index);
808 return ops->enqueue(mp, obj_table, n);
809}
810
819unsigned
820rte_mempool_ops_get_count(const struct rte_mempool *mp);
821
841ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
842 uint32_t obj_num, uint32_t pg_shift,
843 size_t *min_chunk_size, size_t *align);
844
868int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
869 void *vaddr, rte_iova_t iova, size_t len,
871 void *obj_cb_arg);
872
886 struct rte_mempool_info *info);
887
894void
895rte_mempool_ops_free(struct rte_mempool *mp);
896
914int
916 void *pool_config);
917
929
935#define RTE_MEMPOOL_REGISTER_OPS(ops) \
936 RTE_INIT(mp_hdlr_init_##ops) \
937 { \
938 rte_mempool_register_ops(&ops); \
939 }
940
942#define MEMPOOL_REGISTER_OPS(ops) \
943 RTE_DEPRECATED(MEMPOOL_REGISTER_OPS) RTE_MEMPOOL_REGISTER_OPS(ops)
944
950typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
951 void *opaque, void *obj, unsigned obj_idx);
952typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
953
959typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
960 void *opaque, struct rte_mempool_memhdr *memhdr,
961 unsigned mem_idx);
962
969typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
970
1050struct rte_mempool *
1051rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1052 unsigned cache_size, unsigned private_data_size,
1053 rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1054 rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1055 int socket_id, unsigned flags);
1056
1091struct rte_mempool *
1092rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1093 unsigned cache_size, unsigned private_data_size,
1094 int socket_id, unsigned flags);
1105void
1107
1138int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1139 rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1140 void *opaque);
1141
1168int
1170 size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1171 void *opaque);
1172
1187
1202
1219 rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1220
1237 rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1238
1247void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1248
1263struct rte_mempool_cache *
1264rte_mempool_cache_create(uint32_t size, int socket_id);
1265
1272void
1274
1287rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1288{
1289 if (mp->cache_size == 0)
1290 return NULL;
1291
1292 if (lcore_id >= RTE_MAX_LCORE)
1293 return NULL;
1294
1295 rte_mempool_trace_default_cache(mp, lcore_id,
1296 &mp->local_cache[lcore_id]);
1297 return &mp->local_cache[lcore_id];
1298}
1299
1308static __rte_always_inline void
1310 struct rte_mempool *mp)
1311{
1312 if (cache == NULL)
1314 if (cache == NULL || cache->len == 0)
1315 return;
1316 rte_mempool_trace_cache_flush(cache, mp);
1317 rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1318 cache->len = 0;
1319}
1320
1333static __rte_always_inline void
1334rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1335 unsigned int n, struct rte_mempool_cache *cache)
1336{
1337 void **cache_objs;
1338
1339 /* increment stat now, adding in mempool always success */
1340 RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1341 RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1342
1343 /* No cache provided or if put would overflow mem allocated for cache */
1344 if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1345 goto ring_enqueue;
1346
1347 cache_objs = &cache->objs[cache->len];
1348
1349 /*
1350 * The cache follows the following algorithm
1351 * 1. Add the objects to the cache
1352 * 2. Anything greater than the cache min value (if it crosses the
1353 * cache flush threshold) is flushed to the ring.
1354 */
1355
1356 /* Add elements back into the cache */
1357 rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1358
1359 cache->len += n;
1360
1361 if (cache->len >= cache->flushthresh) {
1362 rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1363 cache->len - cache->size);
1364 cache->len = cache->size;
1365 }
1366
1367 return;
1368
1369ring_enqueue:
1370
1371 /* push remaining objects in ring */
1372#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1373 if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1374 rte_panic("cannot put objects in mempool\n");
1375#else
1376 rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1377#endif
1378}
1379
1380
1393static __rte_always_inline void
1394rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1395 unsigned int n, struct rte_mempool_cache *cache)
1396{
1397 rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1398 RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1399 rte_mempool_do_generic_put(mp, obj_table, n, cache);
1400}
1401
1416static __rte_always_inline void
1417rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1418 unsigned int n)
1419{
1420 struct rte_mempool_cache *cache;
1422 rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1423 rte_mempool_generic_put(mp, obj_table, n, cache);
1424}
1425
1438static __rte_always_inline void
1439rte_mempool_put(struct rte_mempool *mp, void *obj)
1440{
1441 rte_mempool_put_bulk(mp, &obj, 1);
1442}
1443
1458static __rte_always_inline int
1459rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1460 unsigned int n, struct rte_mempool_cache *cache)
1461{
1462 int ret;
1463 uint32_t index, len;
1464 void **cache_objs;
1465
1466 /* No cache provided or cannot be satisfied from cache */
1467 if (unlikely(cache == NULL || n >= cache->size))
1468 goto ring_dequeue;
1469
1470 cache_objs = cache->objs;
1471
1472 /* Can this be satisfied from the cache? */
1473 if (cache->len < n) {
1474 /* No. Backfill the cache first, and then fill from it */
1475 uint32_t req = n + (cache->size - cache->len);
1476
1477 /* How many do we require i.e. number to fill the cache + the request */
1478 ret = rte_mempool_ops_dequeue_bulk(mp,
1479 &cache->objs[cache->len], req);
1480 if (unlikely(ret < 0)) {
1481 /*
1482 * In the off chance that we are buffer constrained,
1483 * where we are not able to allocate cache + n, go to
1484 * the ring directly. If that fails, we are truly out of
1485 * buffers.
1486 */
1487 goto ring_dequeue;
1488 }
1489
1490 cache->len += req;
1491 }
1492
1493 /* Now fill in the response ... */
1494 for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1495 *obj_table = cache_objs[len];
1496
1497 cache->len -= n;
1498
1499 RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1500 RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1501
1502 return 0;
1503
1504ring_dequeue:
1505
1506 /* get remaining objects from ring */
1507 ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1508
1509 if (ret < 0) {
1510 RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1511 RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1512 } else {
1513 RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1514 RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1515 }
1516
1517 return ret;
1518}
1519
1540static __rte_always_inline int
1541rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1542 unsigned int n, struct rte_mempool_cache *cache)
1543{
1544 int ret;
1545 ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1546 if (ret == 0)
1547 RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1548 rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1549 return ret;
1550}
1551
1574static __rte_always_inline int
1575rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1576{
1577 struct rte_mempool_cache *cache;
1579 rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1580 return rte_mempool_generic_get(mp, obj_table, n, cache);
1581}
1582
1603static __rte_always_inline int
1604rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1605{
1606 return rte_mempool_get_bulk(mp, obj_p, 1);
1607}
1608
1630static __rte_always_inline int
1632 void **first_obj_table, unsigned int n)
1633{
1634 int ret;
1635
1636 ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1637 if (ret == 0) {
1638 RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1639 RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1640 RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1641 1);
1642 } else {
1643 RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1644 RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1645 }
1646
1647 rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1648 return ret;
1649}
1650
1663unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1664
1677unsigned int
1679
1693static inline int
1695{
1696 return rte_mempool_avail_count(mp) == mp->size;
1697}
1698
1712static inline int
1714{
1715 return rte_mempool_avail_count(mp) == 0;
1716}
1717
1728static inline rte_iova_t
1730{
1731 const struct rte_mempool_objhdr *hdr;
1732 hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1733 sizeof(*hdr));
1734 return hdr->iova;
1735}
1736
1748
1757static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1758{
1759 return (char *)mp +
1760 RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
1761}
1762
1770
1784
1802uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1803 struct rte_mempool_objsz *sz);
1804
1813void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1814 void *arg);
1815
1820int
1821rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1822
1832};
1833
1843typedef void (rte_mempool_event_callback)(
1844 enum rte_mempool_event event,
1845 struct rte_mempool *mp,
1846 void *user_data);
1847
1862__rte_internal
1863int
1864rte_mempool_event_callback_register(rte_mempool_event_callback *func,
1865 void *user_data);
1866
1880__rte_internal
1881int
1882rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
1883 void *user_data);
1884
1885#ifdef __cplusplus
1886}
1887#endif
1888
1889#endif /* _RTE_MEMPOOL_H_ */
#define unlikely(x)
#define __rte_cache_aligned
Definition: rte_common.h:402
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:255
uint64_t rte_iova_t
Definition: rte_common.h:420
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:250
#define RTE_STD_C11
Definition: rte_common.h:42
#define __rte_always_inline
Definition: rte_common.h:228
#define rte_panic(...)
Definition: rte_debug.h:43
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:76
static void * rte_memcpy(void *dst, const void *src, size_t n)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:179
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1575
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1287
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:950
struct rte_mempool * rte_mempool_lookup(const char *name)
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:445
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:357
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1729
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:451
void rte_mempool_free(struct rte_mempool *mp)
rte_mempool_event
Definition: rte_mempool.h:1827
@ RTE_MEMPOOL_EVENT_DESTROY
Definition: rte_mempool.h:1831
@ RTE_MEMPOOL_EVENT_READY
Definition: rte_mempool.h:1829
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:463
void() rte_mempool_populate_obj_cb_t(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:554
int rte_mempool_populate_default(struct rte_mempool *mp)
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1631
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:457
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:440
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1309
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1417
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1604
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1394
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1694
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:435
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1541
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
void rte_mempool_audit(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:423
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:969
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:959
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:488
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1439
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
void rte_mempool_list_dump(FILE *f)
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:678
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1713
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:585
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1757
RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:646
uint32_t flushthresh
Definition: rte_mempool.h:91
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:97
unsigned int contig_block_size
Definition: rte_mempool.h:206
RTE_STAILQ_ENTRY(rte_mempool_memhdr) next
struct rte_mempool * mp
Definition: rte_mempool.h:190
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:194
struct rte_mempool * mp
Definition: rte_mempool.h:145
RTE_STAILQ_ENTRY(rte_mempool_objhdr) next
uint32_t header_size
Definition: rte_mempool.h:105
uint32_t trailer_size
Definition: rte_mempool.h:106
uint32_t total_size
Definition: rte_mempool.h:107
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:695
rte_spinlock_t sl
Definition: rte_mempool.h:690
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:652
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:653
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:656
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:671
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:662
rte_mempool_get_count get_count
Definition: rte_mempool.h:657
rte_mempool_populate_t populate
Definition: rte_mempool.h:667
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:675
rte_mempool_free_t free
Definition: rte_mempool.h:654
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:655
uint32_t nb_mem_chunks
Definition: rte_mempool.h:245
const struct rte_memzone * mz
Definition: rte_mempool.h:220
struct rte_mempool_memhdr_list mem_list
Definition: rte_mempool.h:246
uint32_t populated_size
Definition: rte_mempool.h:243
uint32_t header_size
Definition: rte_mempool.h:228
uint64_t pool_id
Definition: rte_mempool.h:217
int32_t ops_index
Definition: rte_mempool.h:239
void * pool_config
Definition: rte_mempool.h:219
uint32_t trailer_size
Definition: rte_mempool.h:229
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:213
uint32_t size
Definition: rte_mempool.h:223
uint32_t cache_size
Definition: rte_mempool.h:224
unsigned int flags
Definition: rte_mempool.h:221
uint32_t elt_size
Definition: rte_mempool.h:227
unsigned private_data_size
Definition: rte_mempool.h:231
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:241
struct rte_mempool_objhdr_list elt_list
Definition: rte_mempool.h:244
void * pool_data
Definition: rte_mempool.h:216