linux内核内存管理的三个阶段分析（3）

----------------------------------------
硬件：E500v2内核PowerPC ,linux版本:2.6.35
----------------------------------------

3. 全部内存初始化完毕，可以用cache和buddy分配内存。

相关常量定义：
 26 #define NODE_DATA(nid)      (node_data[nid])

 782 extern struct pglist_data contig_page_data;
 783 #define NODE_DATA(nid)      (&contig_page_data)
 784 #define NODE_MEM_MAP(nid)   mem_map

4714 struct pglist_data __refdata contig_page_data = {
4716     .bdata = &bootmem_node_data[0]
4718 };

38 bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;//MAX_NUMNODES=0

 31 typedef struct bootmem_data {
 32     unsigned long node_min_pfn;
 33     unsigned long node_low_pfn;
 34     void *node_bootmem_map;
 35     unsigned long last_end_off;
 36     unsigned long hint_idx;
 37     struct list_head list;
 38 } bootmem_data_t;

 609 typedef struct pglist_data {
 610     struct zone node_zones[MAX_NR_ZONES];
 611     struct zonelist node_zonelists[MAX_ZONELISTS];
 612     int nr_zones;
 613 #ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */
 614     struct page *node_mem_map;
 615 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 616     struct page_cgroup *node_page_cgroup;
 617 #endif
 618 #endif
 619 #ifndef CONFIG_NO_BOOTMEM
 620     struct bootmem_data *bdata;
 621 #endif
 622 #ifdef CONFIG_MEMORY_HOTPLUG
 623     /*
 624      * Must be held any time you expect node_start_pfn, node_present_pages
 625      * or node_spanned_pages stay constant.  Holding this will also
 626      * guarantee that any pfn_valid() stays that way.
 627      *
 628      * Nests above zone->lock and zone->size_seqlock.
 629      */
 630     spinlock_t node_size_lock;
 631 #endif
 632     unsigned long node_start_pfn;
 633     unsigned long node_present_pages; /* total number of physical pages */
 634     unsigned long node_spanned_pages; /* total size of physical page
 635                          range, including holes */
 636     int node_id;
 637     wait_queue_head_t kswapd_wait;
 638     struct task_struct *kswapd;
 639     int kswapd_max_order;
 640 } pg_data_t;

 576 struct zonelist {
 577     struct zonelist_cache *zlcache_ptr;          // NULL or &zlcache = NULL
 578     struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];//MAX_ZONES_PER_ZONELIST=4
 579 #ifdef CONFIG_NUMA
 580     struct zonelist_cache zlcache;               // optional ...
 581 #endif
 582 };

 554 struct zoneref {
 555     struct zone *zone;  /* Pointer to actual zone */
 556     int zone_idx;       /* zone_idx(zoneref->zone) */
 557 };

内存初始化步骤：
1. start_kernel---->setup_arch->pageing_init
                |-->setup_per_cpu_areas
                |-->build_all_zonelists
                |-->mem_init
                 -->setup_per_cpu_pageset


start_kernel---->setup_arch->pageing_init代码如下：

273 /*
274  * paging_init() sets up the page tables - in fact we've already done this.
275  */
276 void __init paging_init(void)
277 {
278        unsigned long total_ram = memblock_phys_mem_size();
279        phys_addr_t top_of_ram = memblock_end_of_DRAM();
280        unsigned long max_zone_pfns[MAX_NR_ZONES];
281
282 #ifdef CONFIG_PPC32
283        unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
284         unsigned long end = __fix_to_virt(FIX_HOLE);
285
286         for (; v < end; v += PAGE_SIZE)
287            map_page(v, 0, 0); /* XXX gross */
288 #endif
289
290 #ifdef CONFIG_HIGHMEM
291        map_page(PKMAP_BASE, 0, 0); /* XXX gross */
292        pkmap_page_table = virt_to_kpte(PKMAP_BASE);
293
294        kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
295         kmap_prot = PAGE_KERNEL;
296 #endif /* CONFIG_HIGHMEM */
297
298        printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%lx\n",
299                       (unsigned long long)top_of_ram, total_ram);
300        printk(KERN_DEBUG "Memory hole size: %ldMB\n",
301                      (long int)((top_of_ram - total_ram) >> 20));
302         memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
303 #ifdef CONFIG_HIGHMEM
304         max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
305         max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
306 #else
307         max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
308 #endif
309         free_area_init_nodes(max_zone_pfns);
310
311         mark_nonram_nosave();
312 }
313 #endif /* ! CONFIG_NEED_MULTIPLE_NODES */

378-280：计算整个内存的大小，此处为：0x8000 0000
282-288: 映射临时映射空间，也称固定映射
290-196：映射高端内存PTE
307：最为重要，该函数清空bitmap区，把所有页设为保留。

                |-->setup_per_cpu_areas
该函数当非SMP情况使用，此处为空。
                |-->build_all_zonelists
函数代码为：

3031 void build_all_zonelists(void *data)
3032 {
3033        set_zonelist_order();
3034
3035         if (system_state == SYSTEM_BOOTING) {
3036                 __build_all_zonelists(NULL);
3037                 mminit_verify_zonelist();
3038         cpuset_init_current_mems_allowed();
3039         } else {
3040            /* we have to stop all cpus to guarantee there is no user
3041               of zonelist */
3042             stop_machine(__build_all_zonelists, data, NULL);
3043            /* cpuset refresh routine should be here */
3044         }
3045         vm_total_pages = nr_free_pagecache_pages();
3046     /*
3047      * Disable grouping by mobility if the number of pages in the
3048      * system is too low to allow the mechanism to work. It would be
3049      * more accurate, but expensive to check per-zone. This check is
3050      * made on memory-hotadd so a system can start with mobility
3051      * disabled and enable it later
3052      */
3053         if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
3054                  page_group_by_mobility_disabled = 1;
3055         else
3056                  page_group_by_mobility_disabled = 0;
3057
3058      printk("Built %i zonelists in %s order, mobility grouping %s.  "
3059                       "Total pages: %ld\n",
3060                 nr_online_nodes,
3061                 zonelist_order_name[current_zonelist_order],
3062                  page_group_by_mobility_disabled ? "off" : "on",
3063                  vm_total_pages);
3064 #ifdef CONFIG_NUMA
3065         printk("Policy zone: %s\n", zone_names[policy_zone]);
3066 #endif
3067 }

函数中：__build_all_zonelists函数建立了zone链表，当一个zone不能满足当前分配时会根据此链表来找到下一个可以分配的zone。当有多个节点时，建立后的结果为：
b2->b1->b0->a2->a1->a0 （b=node1，a=node0，2=highmem, 1=normal, 0=dma)

|-->mem_init函数代码为：

526 static void __init mm_init(void)
527 {
532        page_cgroup_init_flatmem();
533        mem_init();
534        kmem_cache_init();
535        pgtable_cache_init();
536        vmalloc_init();
537 }

532：为空函数
533：函数执行完毕后， mem_init_done = 1;即buddy系统分配内存内可以使用了。
534：用来初始化slab和cache，函数执行完后slab可以使用，可以分配少于1页的内存了。
535：空函数
536：vmalloc空间可以正常使用了。
整个内存的布局为：
Kernel virtual memory layout:
  * 0xfffcf000..0xfffff000  : fixmap
  * 0xff800000..0xffc00000  : highmem PTEs
  * 0xff7d8000..0xff800000  : early ioremap
  * 0xf1000000..0xff7d8000  : vmalloc & ioremap