Chinaunix首页 | 论坛 | 博客

海思了andyhzw.blog.chinaunix.net

首页　| 　博文目录　| 　关于我

andyhzw

  • 博客访问： 6456859
  • 博文数量： 579
  • 博客积分： 1548
  • 博客等级： 上尉
  • 技术积分： 16635
  • 用 户 组： 普通用户
  • 注册时间： 2012-12-12 15:29
个人简介

http://www.csdn.net/ http://www.arm.com/zh/ https://www.kernel.org/ http://www.linuxpk.com/ http://www.51develop.net/ http://linux.chinaitlab.com/ http://www.embeddedlinux.org.cn http://bbs.pediy.com/

文章分类

全部博文（579）

文章存档

2018年（18）

2015年（91）

2014年（159）

2013年（231）

2012年（80）

我的朋友
最近访客
推荐博文
相关博文
linux下C语言实现的内存池

分类： LINUX

2014-05-15 14:07:45

操作系统：ubuntu10.04

前言：
    在通信过程中，无法知道将会接收到的数据的长度，因此开一个固定大小的缓冲区并不合适，开大了，很可能大多数通信都只是几十个自己而已；开小了，又无法处理大数据。因此最好的方法就是创建内存池，根据实际情况，分配合适大小的内存空间。

一，思路


    通过双向链表，管理所有的内存池。


二，实现
    1，内存池的相关信息结构体

点击(此处)折叠或打开

  1. struct pool_head {
  2.     void                 **free_list;    
  3.     struct list_head     list;            /* list of all known pools */
  4.     int32_t                used;            /* how many chunks are currently in use */
  5.     int32_t                allocated;        /* how many chunks have been allocated */
  6.     int32_t                limit;            /* hard limit on the number of chunks */
  7.     int32_t                minavail;        /* how many chunks are expected to be used */
  8.     int32_t                size;            /* chunk size */
  9.     int32_t                flags;            /* MEM_F_* */
  10.     int32_t                users;            /* number of pools sharing this zone */
  11.     int8_t                name[12];        /* name of the pool */
  12. };

    2，具体实现
        a，头文件（pools.h）

点击(此处)折叠或打开

  1. #ifndef __POOLS_H__
  2. #define __POOLS_H__

  4. #ifdef __cplusplus
  5. extern "C" {
  6. #endif

  8. /* Define to prevent recursive inclusion
  9. -------------------------------------*/
  10. #include "types.h"
  11. #include "list.h"


  14. #define MEM_F_SHARED    0x1                /* 标示对应的池允许共用 */


  17. /* 每个池的相关信息 */
  18. struct pool_head {
  19.     void                 **free_list;    
  20.     struct list_head     list;            /* list of all known pools */
  21.     int32_t                used;            /* how many chunks are currently in use */
  22.     int32_t                allocated;        /* how many chunks have been allocated */
  23.     int32_t                limit;            /* hard limit on the number of chunks */
  24.     int32_t                minavail;        /* how many chunks are expected to be used */
  25.     int32_t                size;            /* chunk size */
  26.     int32_t                flags;            /* MEM_F_* */
  27.     int32_t                users;            /* number of pools sharing this zone */
  28.     int8_t                name[12];        /* name of the pool */
  29. };


  32. /* 池创建 */
  33. /* Try to find an existing shared pool with the same characteristics and
  34.  * returns it, otherwise creates this one. NULL is returned if no memory
  35.  * is available for a new creation.
  36.  */
  37. extern     struct pool_head *    pool_create(char *name, uint32_t size, uint32_t flags);


  40. /* 池销毁 */
  41. /*
  42.  * This function destroys a pool by freeing it completely, unless it's still
  43.  * in use. This should be called only under extreme circumstances. It always
  44.  * returns NULL if the resulting pool is empty, easing the clearing of the old
  45.  * pointer, otherwise it returns the pool.
  46.  * .
  47.  */
  48. extern    void*                pool_destroy(struct pool_head *pool);


  51. /* 把池中的空闲的元素都给释放掉 */
  52. /*
  53.  * This function frees whatever can be freed in pool <pool>.
  54.  */
  55. extern     void                 pool_clear(struct pool_head *pool);


  58. /* 把池中非必要的元素给释放掉 */
  59. /*
  60.  * This function frees whatever can be freed in all pools, but respecting
  61.  * the minimum thresholds imposed by owners. It takes care of avoiding
  62.  * recursion because it may be called from a signal handler.
  63.  */
  64. extern    void                 pool_flush_nonessential(void);


  67. /* 动态分配一个 pool 元素大小的内存空间 */
  68. /* Allocate a new entry for pool <pool>, and return it for immediate use.
  69.  * NULL is returned if no memory is available for a new creation.
  70.  */
  71. extern    void *                pool_refill_alloc(struct pool_head *pool);



  75. /*
  76.  * Returns a pointer to type <type> taken from the
  77.  * pool <pool_type> or dynamically allocated. In the
  78.  * first case, <pool_type> is updated to point to the
  79.  * next element in the list.
  80.  */
  81. #define pool_alloc(pool) \
  82. ({ \
  83.         void *__p; \
  84.         if ((__p = (pool)->free_list) == NULL)            \
  85.                 __p = pool_refill_alloc(pool); \
  86.         else { \
  87.                 (pool)->free_list = *(void **)(pool)->free_list;\
  88.         (pool)->used++;                    \
  89.         } \
  90.         __p; \
  91. })



  95. /*
  96.  * Puts a memory area back to the corresponding pool.
  97.  * Items are chained directly through a pointer that
  98.  * is written in the beginning of the memory area, so
  99.  * there's no need for any carrier cell. This implies
  100.  * that each memory area is at least as big as one
  101.  * pointer. Just like with the libc's free(), nothing
  102.  * is done if <ptr> is NULL.
  103.  */
  104. #define pool_free(pool, ptr) \
  105. ({ \
  106.         if ((ptr) != NULL) { \
  107.                 *(void **)(ptr) = (void *)(pool)->free_list;    \
  108.                 (pool)->free_list = (void *)(ptr);    \
  109.                 (pool)->used--;                \
  110.         } \
  111. })



  115. #ifdef __cplusplus
  116. }
  117. #endif

  119. #endif

        b，c文件（pools.c）

点击(此处)折叠或打开

  1. #include "pools.h"
  2. #include "standard.h"


  5. /* 管理所有池的链表头 */
  6. static struct list_head        pools = LIST_HEAD_INIT(pools);




  11. /* 池创建 */
  12. /* Try to find an existing shared pool with the same characteristics and
  13.  * returns it, otherwise creates this one. NULL is returned if no memory
  14.  * is available for a new creation.
  15.  */
  16. struct pool_head *    pool_create(char *name, uint32_t size, uint32_t flags)
  17. {
  18.     struct pool_head *pool;
  19.     struct pool_head *entry;
  20.     struct list_head *start;
  21.     uint32_t         align;

  23.     /* We need to store at least a (void *) in the chunks. Since we know
  24.      * that the malloc() function will never return such a small size,
  25.      * let's round the size up to something slightly bigger, in order to
  26.      * ease merging of entries. Note that the rounding is a power of two.
  27.      */

  29.     align = 16;
  30.     size = (size + align - 1) & -align;

  32.     start = &pools;
  33.     pool = NULL;

  35.     list_for_each_entry(entry, &pools, list) {
  36.         if (entry->size == size) {
  37.             /* either we can share this place and we take it, or
  38.              * we look for a sharable one or for the next position
  39.              * before which we will insert a new one.
  40.              */
  41.             if (flags & entry->flags & MEM_F_SHARED) {
  42.                 /* we can share this one */
  43.                 pool = entry;
  44.                 break;
  45.             }
  46.         }
  47.         else if (entry->size > size) {
  48.             /* insert before this one */
  49.             start = &entry->list;
  50.             break;
  51.         }
  52.     }

  54.     if (!pool) {
  55.         pool = calloc(1, sizeof(*pool));
  56.         if (!pool)
  57.             return NULL;
  58.         if (name)
  59.             strlcpy(pool->name, (int8_t*)name, sizeof(pool->name));
  60.         pool->size = size;
  61.         pool->flags = flags;
  62.         list_add_tail(&pool->list,start);
  63.     }
  64.     pool->users++;
  65.     return pool;
  66. }



  70. /* 池销毁 */
  71. void*                pool_destroy(struct pool_head *pool)
  72. {
  73.     if (pool)
  74.     {
  75.         pool_clear(pool);            // 请看池中的空闲的元素
  76.         if (pool->used)
  77.             return pool;
  78.         pool->users--;
  79.         if (!pool->users)
  80.         {
  81.             list_del(&pool->list);    // 从 pools 链表中删除
  82.             free(pool);                // 把 pool 结构体占用的内存给释放了
  83.         }
  84.     }
  85.     return NULL;
  86. }



  90. /* 把池中的空闲的元素都给释放掉 */
  91. /*
  92.  * This function frees whatever can be freed in pool <pool>.
  93.  */
  94. void                 pool_clear(struct pool_head *pool)
  95. {
  96.     void *temp, *next;
  97.     if (!pool)
  98.         return;

  100.     next = pool->free_list;
  101.     while (next) {
  102.         temp = next;
  103.         next = *(void **)temp;
  104.         pool->allocated--;
  105.         free(temp);
  106.     }
  107.     pool->free_list = next;

  109.     /* here, we should have pool->allocate == pool->used */
  110. }



  114. /* 把池中非必要的元素给释放掉 */
  115. /*
  116.  * This function frees whatever can be freed in all pools, but respecting
  117.  * the minimum thresholds imposed by owners. It takes care of avoiding
  118.  * recursion because it may be called from a signal handler.
  119.  */
  120. void                 pool_flush_nonessential(void)
  121. {
  122.     static int recurse;
  123.     struct pool_head *entry;

  125.     if (recurse++)
  126.         goto out;

  128.     list_for_each_entry(entry, &pools, list) {
  129.         void *temp, *next;
  130.         //qfprintf(stderr, "Flushing pool %s\n", entry->name);
  131.         next = entry->free_list;
  132.         while (next &&
  133.          entry->allocated > entry->minavail &&
  134.          entry->allocated > entry->used) {
  135.             temp = next;
  136.             next = *(void **)temp;
  137.             entry->allocated--;
  138.             free(temp);
  139.         }
  140.         entry->free_list = next;
  141.     }
  142.  out:
  143.     recurse--;
  144. }


  147. /* 动态分配一个 pool 元素大小的内存空间 */
  148. /* Allocate a new entry for pool <pool>, and return it for immediate use.
  149.  * NULL is returned if no memory is available for a new creation. A call
  150.  * to the garbage collector is performed before returning NULL.
  151.  */
  152. void *pool_refill_alloc(struct pool_head *pool)
  153. {
  154.     void *ret;

  156.     if (pool->limit && (pool->allocated >= pool->limit))
  157.         return NULL;
  158.     ret = calloc(1, pool->size);
  159.     if (!ret) {
  160.         pool_flush_nonessential();
  161.         ret = calloc(1, pool->size);
  162.         if (!ret)
  163.             return NULL;
  164.     }

  166.     pool->allocated++;
  167.     pool->used++;
  168.     return ret;
  169. }


  172. /* 销毁所有的池 */
  173. int32_t                dump_pools(void)
  174. {
  175.     int32_t        ret = OPER_OK;

  177.     return ret;
  178. }

        c，辅助文件（standard.c）

点击(此处)折叠或打开

  1. /*
  2.  * copies at most <size-1> chars from <src> to <dst>. Last char is always
  3.  * set to 0, unless <size> is 0. The number of chars copied is returned
  4.  * (excluding the terminating zero).
  5.  * This code has been optimized for size and speed : on x86, it's 45 bytes
  6.  * long, uses only registers, and consumes only 4 cycles per char.
  7.  */
  8. int32_t    strlcpy(int8_t*dst, const int8_t*src, int32_t size)
  9. {
  10.     int8_t *orig = dst;
  11.     if (size)
  12.     {
  13.         while (--size && (*dst = *src))
  14.         {
  15.             src++; dst++;
  16.         }
  17.         *dst = 0;
  18.     }
  19.     return dst - orig;
  20. }

        d，辅助文件（types.h）

点击(此处)折叠或打开

  1. #ifndef __TYPES_H__
  2. #define __TYPES_H__

  4. #ifdef __cplusplus
  5. extern "C" {
  6. #endif

  8. /* Define to prevent recursive inclusion
  9. -------------------------------------*/
  10. #include <stdio.h>     // 标准输入输出定义
  11. #include <stdlib.h>     // 标准函数库定义
  12. #include <string.h>             // memset
  13. #include <unistd.h>     // Unix标准函数定义，read,write...
  14. #include <sys/types.h>
  15. #include <sys/stat.h>
  16. #include <fcntl.h> // 文件控制定义
  17. #include <termios.h>     // POSIX中断控制定义
  18. #include <errno.h>     // 错误号定义
  19. #include <pthread.h>        // pthread_t,pthread_create...
  20. #include "error.h"
  21. #include "debug.h"

  23. /* 类型定义 */
  24. typedef signed        char        int8_t;
  25. typedef unsigned     char         uint8_t;

  27. typedef signed        short         int16_t;
  28. typedef unsigned     short         uint16_t;

  30. typedef signed        int            int32_t;
  31. typedef unsigned     int         uint32_t;

  33. typedef signed        long long     int64_t;
  34. typedef unsigned long long         uint64_t;


  37. #define    BUFFER_SIZE                256

  39. /* 1，COM，串口相关*/
  40. #define    COM_TYPE_UPPER_DEVICE    1
  41. #define    COM_TYPE_LOWER_DEVICE    2

  43. #define    COM_BUFFER_SIZE            (BUFFER_SIZE)


  46. /* 2，pools，池相关 */



  50. /* 3，命令相关*/
  51. #define    CMD_DATA_LEN_MAX        (BUFFER_SIZE)




  56. #ifdef __cplusplus
  57. }
  58. #endif

  60. #endif



三，实例
    1，头文件（command.h）

点击(此处)折叠或打开

  1. #ifndef __COMMAND_H__
  2. #define __COMMAND_H__

  4. #ifdef __cplusplus
  5. extern "C" {
  6. #endif

  8. /* Define to prevent recursive inclusion
  9. -------------------------------------*/
  10. #include "types.h"


  13. /* 接收到的命令的来源等 */
  14. typedef    struct _cmd
  15. {
  16.     int32_t        fd;
  17.     pthread_t    id;
  18.     void*        data;
  19. }cmd_t;



  23. // 创建内存池
  24. extern    int32_t        cmd_pool_create(void);


  27. // 释放内存池
  28. extern    int32_t        cmd_pool_destroy(void);


  31. // 申请命令内存块，用以保存命令数据
  32. extern    int32_t        cmd_alloc(cmd_t    **param);

  34. // 释放命令内存块
  35. extern    int32_t        cmd_free(cmd_t    *param);

  37. #ifdef DEBUG_POOL
  38. extern    void            cmd_pool_info(void);
  39. #endif


  42. #ifdef __cplusplus
  43. }
  44. #endif

  46. #endif

    2，c文件（command.c）

点击(此处)折叠或打开

  1. #include "command.h"
  2. #include "pools.h"


  5. static    struct    pool_head        *cmd_head_pool = NULL;
  6. static    struct    pool_head        *cmd_data_pool = NULL;


  9. // 创建内存池
  10. int32_t        cmd_pool_create(void)
  11. {
  12.     int32_t        ret = OPER_OK;
  13.     
  14.     if (cmd_head_pool == NULL)
  15.     {
  16.         if((cmd_head_pool = pool_create("cmd_head", sizeof(cmd_t), MEM_F_SHARED)) == NULL)
  17.         {
  18.             ret = -POOL_CREATE_ERROR;
  19.         }
  20.         else
  21.         {
  22.             if (cmd_data_pool == NULL)
  23.                 if((cmd_data_pool = pool_create("cmd_data", CMD_DATA_LEN_MAX, MEM_F_SHARED)) == NULL)
  24.                 {
  25.                     cmd_pool_destroy();        
  26.                     ret = -POOL_CREATE_ERROR;
  27.                 }
  28.         }
  29.     }

  31. #ifdef DEBUG_POOL
  32.     cmd_pool_info();
  33. #endif

  35.     return ret;
  36. }


  39. #ifdef DEBUG_POOL
  40. void        cmd_pool_info(void)
  41. {
  42.     struct    pool_head        *entry = cmd_head_pool;
  43.     printf("pool %s (%d bytes) : %d allocated (%u bytes), %d used\n",entry->name, entry->size, entry->allocated,entry->size * entry->allocated, entry->used);

  45.     entry = cmd_data_pool;
  46.     printf("pool %s (%d bytes) : %d allocated (%u bytes), %d used\n",entry->name, entry->size, entry->allocated,entry->size * entry->allocated, entry->used);
  47. }
  48. #endif


  51. // 释放内存池
  52. int32_t        cmd_pool_destroy(void)
  53. {
  54.     int32_t    ret = OPER_OK;

  56. #ifdef DEBUG_POOL
  57.     cmd_pool_info();
  58. #endif

  60.     if(cmd_head_pool != NULL)
  61.     {
  62.         if(NULL != pool_destroy(cmd_head_pool))
  63.         {
  64.             ret = -POOL_DESTROY_ERROR;
  65.         }
  66.         else
  67.         {
  68.             if(cmd_data_pool != NULL)
  69.                 if(NULL != pool_destroy(cmd_data_pool))
  70.                     ret = -POOL_DESTROY_ERROR;
  71.         }
  72.     }

  74.     return ret;
  75. }


  78. // 申请命令内存块，用以保存命令数据
  79. int32_t        cmd_alloc(cmd_t    **param)
  80. {
  81.     int32_t        ret = OPER_OK;

  83.     if((*param = (cmd_t*)pool_alloc(cmd_head_pool)) == NULL)
  84.     {
  85.         ret = -CMD_POOL_ALLOC_ERROR;
  86.     }
  87.     else
  88.     {
  89.         memset(*param,0,sizeof(cmd_t));
  90.         
  91.         if(((*param)->data = pool_alloc(cmd_data_pool)) == NULL)
  92.         {
  93.             cmd_free(*param);
  94.             ret = -CMD_POOL_ALLOC_ERROR;
  95.         }
  96.     }

  98. #ifdef DEBUG_POOL
  99.     cmd_pool_info();
  100. #endif
  101.     
  102.     return ret;
  103. }


  106. // 释放命令内存块
  107. int32_t        cmd_free(cmd_t    *param)
  108. {
  109.     if(param->data != NULL)
  110.     {
  111.         pool_free(cmd_data_pool,param->data);
  112.         param->data = NULL;
  113.     }
  114.     
  115.     if(param != NULL)
  116.     {
  117.         pool_free(cmd_head_pool,param);
  118.         param = NULL;
  119.     }
  120.     
  121. #ifdef DEBUG_POOL
  122.     cmd_pool_info();
  123. #endif
  124.     return OPER_OK;
  125. }

        c，辅助文件（test.c）

点击(此处)折叠或打开

  1. /*
  2.  * cmd pool begin
  3.  */
  4. static    void    cmd_pool_oper(void)
  5. {
  6.     int32_t        ret = OPER_OK;

  8.     printf("command pool test!\n");

  10.     if((ret = cmd_pool_create()) != OPER_OK)
  11.     {
  12.         printf("Create command pool fail!\n");
  13.     }
  14.     else
  15.     {
  16.         cmd_t*    cmd_buf[5];
  17.         int32_t    i = 0;
  18.         int32_t    count = 0;
  19.         
  20.         printf("Create command pool success!!!\n");

  22.         memset(cmd_buf,0,sizeof(cmd_buf));

  24.         for(i = 0; i < 5; i++)
  25.         {
  26.             printf("    alloc \n");
  27.             if(cmd_alloc(&cmd_buf[i]) != OPER_OK)
  28.             {
  29.                 printf("Alloc buffer fail : %d\n",i);
  30.                 count = i;
  31.                 break;
  32.             }
  33.             cmd_buf[i]->fd    = i+1;
  34.             strcpy((char*)cmd_buf[i]->data,"hello");
  35.         }

  37.         printf("Alloc complete success!\n");
  38. //        if(i >= 5)    count = i;
  39.         
  40.         for(i = 0 ; i < 5; i++)
  41.         {
  42.             printf("command %d fd : %d,data : %s\n",(i+1),cmd_buf[i]->fd,(char*)cmd_buf[i]->data);
  43.             cmd_free(cmd_buf[i]);
  44.         }

  46.         if((ret = cmd_pool_destroy()) != OPER_OK)
  47.             printf("command pool destroy fail, still in use\n");
  48.         else
  49.             printf("command pool destroy success!\n");
  50.     }
  51.     
  52. }



  56. void    test_cmd_pool()
  57. {
  58.     cmd_pool_oper();
  59. }

  61. /*
  62.  * cmd pool end
  63.  */

        d，测试结果
        


四，参考文件
1，《haproxy-1.5》源码
2，linux下双向链表的实现




阅读(25833) | 评论(3) | 转发(16) |
0

上一篇：Linux 程序内存空间布局

下一篇：linux下C语言实现多线程通信—环形缓冲区,可用于生产者(producer)/消费者(consumer)

给主人留下些什么吧！~~

erikaemma2017-09-10 19:50:37

请问能否提供完整的源代码呢？

回复 | 举报

Dafanzi2016-12-05 17:07:35

大佬 希望你能看到我的消息。你提供的源代码中，“list.h”没提供啊。我想看完整的，能发我邮箱中来吗？
brottfan@163.com

回复 | 举报

laotian2014-05-17 14:56:31

memcached 不就是这样实现的？

回复 | 举报
关于我们 | 关于IT168 | 联系方式 | 广告合作 | 法律声明 | 免费注册

Copyright 2001-2010 ChinaUnix.net All Rights Reserved 北京皓辰网域网络信息技术有限公司. 版权所有

感谢所有关心和支持过ChinaUnix的朋友们

16024965号-6