Module space空间不足导致insmod失败实例

在arm平台下移植Atheros Wifi驱动时遇到insmod umac.ko失败的问题；

ls -l ./drivers/umac.ko  -h

-rw-r--r--    1 root     root        2.7M Sep  8 06:47 ./drivers/umac.ko

insmod ./umac.ko

[ 2072.000717] vmap allocation for size 458752 failed: use vmalloc= to increase size.

[ 2072.008447] vmalloc: allocation failure: 454123 bytes

[ 2072.012855] insmod: page allocation failure: order:0, mode:0xd0

[ 2072.018885] CPU: 0 PID: 2173 Comm: insmod Tainted: P             3.14.43_cig001 #6

[ 2072.026369] [] (unwind_backtrace) from [] (show_stack+0x10/0x14)

[ 2072.034247] [] (show_stack) from [] (dump_stack+0x88/0xcc)

[ 2072.041370] [] (dump_stack) from [] (warn_alloc_failed+0xd0/0xfc)

[ 2072.049206] [] (warn_alloc_failed) from [] (__vmalloc_node_range+0x1b0/0x1d4)

[ 2072.057939] [] (__vmalloc_node_range) from [] (module_alloc+0x30/0x44)

[ 2072.066266] [] (module_alloc) from [] (module_alloc_update_bounds+0xc/0x5c)

[ 2072.074930] [] (module_alloc_update_bounds) from [] (load_module+0x678/0x1940)

[ 2072.083916] [] (load_module) from [] (SyS_init_module+0xc0/0xd8)

[ 2072.091602] [] (SyS_init_module) from [] (ret_fast_syscall+0x0/0x30)

[ 2072.099663] Mem-info:

[ 2072.101819] Normal per-cpu:

[ 2072.104602] CPU    0: hi:   90, btch:  15 usd:  27

[ 2072.109556] CPU    1: hi:   90, btch:  15 usd:  83

[ 2072.114153] CPU    2: hi:   90, btch:  15 usd:  10

[ 2072.119006] CPU    3: hi:   90, btch:  15 usd:  36

[ 2072.123746] active_anon:1525 inactive_anon:16 isolated_anon:0

[ 2072.123746]  active_file:4500 inactive_file:3187 isolated_file:0

[ 2072.123746]  unevictable:0 dirty:0 writeback:0 unstable:0

[ 2072.123746]  free:40864 slab_reclaimable:813 slab_unreclaimable:4219

[ 2072.123746]  mapped:523 shmem:78 pagetables:95 bounce:0

[ 2072.123746]  free_cma:0

[ 2072.154969] Normal free:163456kB min:1944kB low:2428kB high:2916kB active_anon:6100kB inactive_anon:64kB active_file:18000kB inactive_file:12748kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:245760kB managed:236872kB mlocked:0kB dirty:0kB writeback:0kB mapped:2092kB shmem:312kB slab_reclaimable:3252kB slab_unreclaimable:16876kB kernel_stack:608kB pagetables:380kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no

[ 2072.196622] lowmem_reserve[]: 0 0 0

[ 2072.199993] Normal: 0*4kB 8*8kB (UEM) 3*16kB (EM) 1*32kB (U) 2*64kB (UE) 3*128kB (UEM) 2*256kB (UM) 3*512kB (UEM) 1*1024kB (U) 2*2048kB (UM) 38*4096kB (MR) = 163472kB

[ 2072.214838] 7765 total pagecache pages

[ 2072.218663] 0 pages in swap cache

[ 2072.221877] Swap cache stats: add 0, delete 0, find 0/0

[ 2072.227075] Free swap  = 0kB

[ 2072.230017] Total swap = 0kB

[ 2072.242807] 65536 pages of RAM

[ 2072.244851] 41172 free pages

[ 2072.247715] 6318 reserved pages

[ 2072.250933] 2005 slab pages

[ 2072.253618] 3711 pages shared

[ 2072.256567] 0 pages swap cached

umac本身有2.7M，比较大；从出现的堆栈看问题出在加载阶段，与ko本身的运行没有关系；

insmod命令实质调用kmodloader；

root@OpenWrt:/sbin# ls -l /usr/sbin/insmod

lrwxrwxrwx    1 root     root            21 Sep  4 07:32 /usr/sbin/insmod -> ../../sbin/kmodloader

模块加载的内核态实现，由系统调用SyS_init_module开始，其包裹的load_module进行实质的空间分配和模块加载；

static int load_module(struct load_info *info, const char __user *uargs,

                                   int flags)

{

              struct module *mod;

              long err;

 

              err = module_sig_check(info);          //若module存在signature，则进行检查

              if (err)

                            goto free_copy;

 

              err = elf_header_check(info);           //elf文件格式检查

              if (err)

                            goto free_copy;

 

              /* Figure out module layout, and allocate all the memory. */

              mod = layout_and_allocate(info, flags);

              if (IS_ERR(mod)) {

                            err = PTR_ERR(mod);

                            goto free_copy;

              }

              ……

}

static struct module *layout_and_allocate(struct load_info *info, int flags)

{

              /* Module within temporary copy. */

              struct module *mod;

              int err;

 

              mod = setup_load_info(info, flags);               //记录模块基本信息，以便后续调用；

              if (IS_ERR(mod))

                            return mod;

 

              err = check_modinfo(mod, info, flags);

              if (err)

                            return ERR_PTR(err);

 

              /* Allow arches to frob section contents and sizes.  */

              err = module_frob_arch_sections(info->hdr, info->sechdrs,

                                                                      info->secstrings, mod);

              if (err < 0)

                            return ERR_PTR(err);

 

              /* We will do a special allocation for per-cpu sections later. */

              info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;

 

              /* Determine total sizes, and put offsets in sh_entsize.  For now

                 this is done generically; there doesn't appear to be any

                 special cases for the architectures. */ 

              layout_sections(mod, info);

              layout_symtab(mod, info);

 

              /* Allocate and move to the final place */

              err = move_module(mod, info);       //在该函数内分配地址空间；

              if (err)

                            return ERR_PTR(err);

 

              /* Module has been copied to its final place now: return it. */

              mod = (void *)info->sechdrs[info->index.mod].sh_addr;

              kmemleak_load_module(mod, info);

              return mod;

}

static int move_module(struct module *mod, struct load_info *info)

{

              int i;

              void *ptr;

 

              /* Do the allocs. */

              ptr = module_alloc_update_bounds(mod->core_size);

              ……

}

static void *module_alloc_update_bounds(unsigned long size)

{

              void *ret = module_alloc(size);

 

              if (ret) {

                            mutex_lock(&module_mutex);

                            /* Update module bounds. */

                            if ((unsigned long)ret < module_addr_min)

                                          module_addr_min = (unsigned long)ret;

                            if ((unsigned long)ret + size > module_addr_max)

                                          module_addr_max = (unsigned long)ret + size;

                            mutex_unlock(&module_mutex);

              }

              return ret;

}

void *module_alloc(unsigned long size)

{

              return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,

                                                        GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE,

                                                        __builtin_return_address(0));

}

从上面函数可以看到module加载存在特定的地址范围，[MODULES_VADDR, MODULES_END];

这两个宏定义在；linux-3.14.43/arch/arm/include/asm/memory.h，堆栈信息中提示是空间分配失败导致的，那么这段地址范围有多大呐

[wms@localhost linux-3.14.43]$ cat .config | grep CONFIG_PAGE_OFFSET

CONFIG_PAGE_OFFSET=0xC0000000          //3G

#define PAGE_OFFSET                         UL(CONFIG_PAGE_OFFSET)               //3G

 

/*

 * TASK_SIZE - the maximum size of a user space task.   //用户空间最大地址范围；

 * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area

 */

#define TASK_SIZE                   (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M))

#define TASK_UNMAPPED_BASE     ALIGN(TASK_SIZE / 3, SZ_16M)

 

/*

 * The module space lives between the addresses given by TASK_SIZE

 * and PAGE_OFFSET - it must be within 32MB of the kernel text.           //注意module space 最大空间为32MB

 */

#ifndef CONFIG_THUMB2_KERNEL

#define MODULES_VADDR                (PAGE_OFFSET - SZ_16M)

#else

/* smaller range for Thumb-2 symbols relocation (2^24)*/

#define MODULES_VADDR                (PAGE_OFFSET - SZ_8M)

#endif

 

#if TASK_SIZE > MODULES_VADDR                            /*若用户态最大地址与module地址存在冲突，编译时报警*/

#error Top of user space clashes with start of module space

#endif

 

/*

 * The highmem pkmap virtual space shares the end of the module area.

 */

#ifdef CONFIG_HIGHMEM

#define MODULES_END                      (PAGE_OFFSET - PMD_SIZE)

#else

#define MODULES_END                      (PAGE_OFFSET)

#endif

 

#define PMD_SHIFT                21

#define PGDIR_SHIFT                           21

 

#define PMD_SIZE                   (1UL << PMD_SHIFT)                            //2M

从上面宏定义中可以看出；

用户态最大地址TASK_SIZE为：3G-16M;

Module分配空间地址范围为：[3G-16M, 3G-2M];

必须保证TASK_SIZE与MODULES_VADDR不重叠；

针对上面的问题，我们通过扩展Module的范围来解决；

[wms@localhost linux-3.14.43]$ git diff ./arch/arm/include/

diff --git a/linux-3.14.43/arch/arm/include/asm/memory.h b/linux-3.14.43/arch/arm/include/asm/memory.h

old mode 100644

new mode 100755

index 4afb376..deca0b2

--- a/linux-3.14.43/arch/arm/include/asm/memory.h

+++ b/linux-3.14.43/arch/arm/include/asm/memory.h

@@ -39,7 +39,8 @@

  * TASK_SIZE - the maximum size of a user space task.

  * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area

  */

-#define TASK_SIZE              (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M))

+#define TASK_SIZE              (UL(CONFIG_PAGE_OFFSET) - UL(SZ_32M))

 #define TASK_UNMAPPED_BASE     ALIGN(TASK_SIZE / 3, SZ_16M)

 

 /*

@@ -52,7 +53,8 @@

  * and PAGE_OFFSET - it must be within 32MB of the kernel text.

  */

 #ifndef CONFIG_THUMB2_KERNEL

-#define MODULES_VADDR          (PAGE_OFFSET - SZ_16M)

+#define MODULES_VADDR          (PAGE_OFFSET - SZ_32M)

 #else

 /* smaller range for Thumb-2 symbols relocation (2^24)*/

 #define MODULES_VADDR          (PAGE_OFFSET - SZ_8M)