arch/x86/mm/init.c

   1 #include <linux/gfp.h>
   2 #include <linux/initrd.h>
   3 #include <linux/ioport.h>
   4 #include <linux/swap.h>
   5 #include <linux/memblock.h>
   6 #include <linux/bootmem.h>      /* for max_low_pfn */
   7
   8 #include <asm/cacheflush.h>
   9 #include <asm/e820.h>
  10 #include <asm/init.h>
  11 #include <asm/page.h>
  12 #include <asm/page_types.h>
  13 #include <asm/sections.h>
  14 #include <asm/setup.h>
  15 #include <asm/tlbflush.h>
  16 #include <asm/tlb.h>
  17 #include <asm/proto.h>
  18 #include <asm/dma.h>            /* for MAX_DMA_PFN */
  19
  20 #include "mm_internal.h"
  21
  22 unsigned long __initdata pgt_buf_start;
  23 unsigned long __meminitdata pgt_buf_end;
  24 unsigned long __meminitdata pgt_buf_top;
  25
  26 __ref void *alloc_low_page(void)
  27 {
  28         unsigned long pfn;
  29         void *adr;
  30
  31 #ifdef CONFIG_X86_64
  32         if (after_bootmem) {
  33                 adr = (void *)get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
  34
  35                 return adr;
  36         }
  37 #endif
  38
  39         if ((pgt_buf_end + 1) >= pgt_buf_top) {
  40                 unsigned long ret;
  41                 if (min_pfn_mapped >= max_pfn_mapped)
  42                         panic("alloc_low_page: ran out of memory");
  43                 ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
  44                                         max_pfn_mapped << PAGE_SHIFT,
  45                                         PAGE_SIZE, PAGE_SIZE);
  46                 if (!ret)
  47                         panic("alloc_low_page: can not alloc memory");
  48                 memblock_reserve(ret, PAGE_SIZE);
  49                 pfn = ret >> PAGE_SHIFT;
  50         } else
  51                 pfn = pgt_buf_end++;
  52
  53         adr = __va(pfn * PAGE_SIZE);
  54         clear_page(adr);
  55         return adr;
  56 }
  57
  58 /* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */
  59 #define INIT_PGT_BUF_SIZE       (5 * PAGE_SIZE)
  60 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
  61 void  __init early_alloc_pgt_buf(void)
  62 {
  63         unsigned long tables = INIT_PGT_BUF_SIZE;
  64         phys_addr_t base;
  65
  66         base = __pa(extend_brk(tables, PAGE_SIZE));
  67
  68         pgt_buf_start = base >> PAGE_SHIFT;
  69         pgt_buf_end = pgt_buf_start;
  70         pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
  71 }
  72
  73 int after_bootmem;
  74
  75 int direct_gbpages
  76 #ifdef CONFIG_DIRECT_GBPAGES
  77                                 = 1
  78 #endif
  79 ;
  80
  81 struct map_range {
  82         unsigned long start;
  83         unsigned long end;
  84         unsigned page_size_mask;
  85 };
  86
  87 static int page_size_mask;
  88
  89 static void __init probe_page_size_mask(void)
  90 {
  91 #if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
  92         /*
  93          * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
  94          * This will simplify cpa(), which otherwise needs to support splitting
  95          * large pages into small in interrupt context, etc.
  96          */
  97         if (direct_gbpages)
  98                 page_size_mask |= 1 << PG_LEVEL_1G;
  99         if (cpu_has_pse)
 100                 page_size_mask |= 1 << PG_LEVEL_2M;
 101 #endif
 102
 103         /* Enable PSE if available */
 104         if (cpu_has_pse)
 105                 set_in_cr4(X86_CR4_PSE);
 106
 107         /* Enable PGE if available */
 108         if (cpu_has_pge) {
 109                 set_in_cr4(X86_CR4_PGE);
 110                 __supported_pte_mask |= _PAGE_GLOBAL;
 111         }
 112 }
 113 void __init native_pagetable_reserve(u64 start, u64 end)
 114 {
 115         memblock_reserve(start, end - start);
 116 }
 117
 118 #ifdef CONFIG_X86_32
 119 #define NR_RANGE_MR 3
 120 #else /* CONFIG_X86_64 */
 121 #define NR_RANGE_MR 5
 122 #endif
 123
 124 static int __meminit save_mr(struct map_range *mr, int nr_range,
 125                              unsigned long start_pfn, unsigned long end_pfn,
 126                              unsigned long page_size_mask)
 127 {
 128         if (start_pfn < end_pfn) {
 129                 if (nr_range >= NR_RANGE_MR)
 130                         panic("run out of range for init_memory_mapping\n");
 131                 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
 132                 mr[nr_range].end   = end_pfn<<PAGE_SHIFT;
 133                 mr[nr_range].page_size_mask = page_size_mask;
 134                 nr_range++;
 135         }
 136
 137         return nr_range;
 138 }
 139
 140 /*
 141  * adjust the page_size_mask for small range to go with
 142  *      big page size instead small one if nearby are ram too.
 143  */
 144 static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
 145                                                          int nr_range)
 146 {
 147         int i;
 148
 149         for (i = 0; i < nr_range; i++) {
 150                 if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
 151                     !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
 152                         unsigned long start = round_down(mr[i].start, PMD_SIZE);
 153                         unsigned long end = round_up(mr[i].end, PMD_SIZE);
 154
 155 #ifdef CONFIG_X86_32
 156                         if ((end >> PAGE_SHIFT) > max_low_pfn)
 157                                 continue;
 158 #endif
 159
 160                         if (memblock_is_region_memory(start, end - start))
 161                                 mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
 162                 }
 163                 if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
 164                     !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
 165                         unsigned long start = round_down(mr[i].start, PUD_SIZE);
 166                         unsigned long end = round_up(mr[i].end, PUD_SIZE);
 167
 168                         if (memblock_is_region_memory(start, end - start))
 169                                 mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
 170                 }
 171         }
 172 }
 173
 174 static int __meminit split_mem_range(struct map_range *mr, int nr_range,
 175                                      unsigned long start,
 176                                      unsigned long end)
 177 {
 178         unsigned long start_pfn, end_pfn;
 179         unsigned long pos;
 180         int i;
 181
 182         /* head if not big page alignment ? */
 183         start_pfn = start >> PAGE_SHIFT;
 184         pos = start_pfn << PAGE_SHIFT;
 185 #ifdef CONFIG_X86_32
 186         /*
 187          * Don't use a large page for the first 2/4MB of memory
 188          * because there are often fixed size MTRRs in there
 189          * and overlapping MTRRs into large pages can cause
 190          * slowdowns.
 191          */
 192         if (pos == 0)
 193                 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
 194         else
 195                 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
 196                                  << (PMD_SHIFT - PAGE_SHIFT);
 197 #else /* CONFIG_X86_64 */
 198         end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
 199                         << (PMD_SHIFT - PAGE_SHIFT);
 200 #endif
 201         if (end_pfn > (end >> PAGE_SHIFT))
 202                 end_pfn = end >> PAGE_SHIFT;
 203         if (start_pfn < end_pfn) {
 204                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 205                 pos = end_pfn << PAGE_SHIFT;
 206         }
 207
 208         /* big page (2M) range */
 209         start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
 210                          << (PMD_SHIFT - PAGE_SHIFT);
 211 #ifdef CONFIG_X86_32
 212         end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
 213 #else /* CONFIG_X86_64 */
 214         end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
 215                          << (PUD_SHIFT - PAGE_SHIFT);
 216         if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
 217                 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
 218 #endif
 219
 220         if (start_pfn < end_pfn) {
 221                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 222                                 page_size_mask & (1<<PG_LEVEL_2M));
 223                 pos = end_pfn << PAGE_SHIFT;
 224         }
 225
 226 #ifdef CONFIG_X86_64
 227         /* big page (1G) range */
 228         start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
 229                          << (PUD_SHIFT - PAGE_SHIFT);
 230         end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
 231         if (start_pfn < end_pfn) {
 232                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 233                                 page_size_mask &
 234                                  ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
 235                 pos = end_pfn << PAGE_SHIFT;
 236         }
 237
 238         /* tail is not big page (1G) alignment */
 239         start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
 240                          << (PMD_SHIFT - PAGE_SHIFT);
 241         end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
 242         if (start_pfn < end_pfn) {
 243                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 244                                 page_size_mask & (1<<PG_LEVEL_2M));
 245                 pos = end_pfn << PAGE_SHIFT;
 246         }
 247 #endif
 248
 249         /* tail is not big page (2M) alignment */
 250         start_pfn = pos>>PAGE_SHIFT;
 251         end_pfn = end>>PAGE_SHIFT;
 252         nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 253
 254         /* try to merge same page size and continuous */
 255         for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
 256                 unsigned long old_start;
 257                 if (mr[i].end != mr[i+1].start ||
 258                     mr[i].page_size_mask != mr[i+1].page_size_mask)
 259                         continue;
 260                 /* move it */
 261                 old_start = mr[i].start;
 262                 memmove(&mr[i], &mr[i+1],
 263                         (nr_range - 1 - i) * sizeof(struct map_range));
 264                 mr[i--].start = old_start;
 265                 nr_range--;
 266         }
 267
 268         if (!after_bootmem)
 269                 adjust_range_page_size_mask(mr, nr_range);
 270
 271         for (i = 0; i < nr_range; i++)
 272                 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
 273                                 mr[i].start, mr[i].end - 1,
 274                         (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
 275                          (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
 276
 277         return nr_range;
 278 }
 279
 280 static struct range pfn_mapped[E820_X_MAX];
 281 static int nr_pfn_mapped;
 282
 283 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
 284 {
 285         nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
 286                                              nr_pfn_mapped, start_pfn, end_pfn);
 287         nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
 288
 289         max_pfn_mapped = max(max_pfn_mapped, end_pfn);
 290
 291         if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
 292                 max_low_pfn_mapped = max(max_low_pfn_mapped,
 293                                          min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
 294 }
 295
 296 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
 297 {
 298         int i;
 299
 300         for (i = 0; i < nr_pfn_mapped; i++)
 301                 if ((start_pfn >= pfn_mapped[i].start) &&
 302                     (end_pfn <= pfn_mapped[i].end))
 303                         return true;
 304
 305         return false;
 306 }
 307
 308 /*
 309  * Setup the direct mapping of the physical memory at PAGE_OFFSET.
 310  * This runs before bootmem is initialized and gets pages directly from
 311  * the physical memory. To access them they are temporarily mapped.
 312  */
 313 unsigned long __init_refok init_memory_mapping(unsigned long start,
 314                                                unsigned long end)
 315 {
 316         struct map_range mr[NR_RANGE_MR];
 317         unsigned long ret = 0;
 318         int nr_range, i;
 319
 320         pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
 321                start, end - 1);
 322
 323         memset(mr, 0, sizeof(mr));
 324         nr_range = split_mem_range(mr, 0, start, end);
 325
 326         for (i = 0; i < nr_range; i++)
 327                 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
 328                                                    mr[i].page_size_mask);
 329
 330 #ifdef CONFIG_X86_32
 331         early_ioremap_page_table_range_init();
 332
 333         load_cr3(swapper_pg_dir);
 334 #endif
 335
 336         __flush_tlb_all();
 337
 338         add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
 339
 340         return ret >> PAGE_SHIFT;
 341 }
 342
 343 /*
 344  * would have hole in the middle or ends, and only ram parts will be mapped.
 345  */
 346 static unsigned long __init init_range_memory_mapping(
 347                                            unsigned long range_start,
 348                                            unsigned long range_end)
 349 {
 350         unsigned long start_pfn, end_pfn;
 351         unsigned long mapped_ram_size = 0;
 352         int i;
 353
 354         for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
 355                 u64 start = (u64)start_pfn << PAGE_SHIFT;
 356                 u64 end = (u64)end_pfn << PAGE_SHIFT;
 357
 358                 if (end <= range_start)
 359                         continue;
 360
 361                 if (start < range_start)
 362                         start = range_start;
 363
 364                 if (start >= range_end)
 365                         continue;
 366
 367                 if (end > range_end)
 368                         end = range_end;
 369
 370                 init_memory_mapping(start, end);
 371
 372                 mapped_ram_size += end - start;
 373         }
 374
 375         return mapped_ram_size;
 376 }
 377
 378 /* (PUD_SHIFT-PMD_SHIFT)/2 */
 379 #define STEP_SIZE_SHIFT 5
 380 void __init init_mem_mapping(void)
 381 {
 382         unsigned long end, real_end, start, last_start;
 383         unsigned long step_size;
 384         unsigned long addr;
 385         unsigned long mapped_ram_size = 0;
 386         unsigned long new_mapped_ram_size;
 387
 388         probe_page_size_mask();
 389
 390 #ifdef CONFIG_X86_64
 391         end = max_pfn << PAGE_SHIFT;
 392 #else
 393         end = max_low_pfn << PAGE_SHIFT;
 394 #endif
 395
 396         /* the ISA range is always mapped regardless of memory holes */
 397         init_memory_mapping(0, ISA_END_ADDRESS);
 398
 399         /* xen has big range in reserved near end of ram, skip it at first */
 400         addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE,
 401                          PAGE_SIZE);
 402         real_end = addr + PMD_SIZE;
 403
 404         /* step_size need to be small so pgt_buf from BRK could cover it */
 405         step_size = PMD_SIZE;
 406         max_pfn_mapped = 0; /* will get exact value next */
 407         min_pfn_mapped = real_end >> PAGE_SHIFT;
 408         last_start = start = real_end;
 409         while (last_start > ISA_END_ADDRESS) {
 410                 if (last_start > step_size) {
 411                         start = round_down(last_start - 1, step_size);
 412                         if (start < ISA_END_ADDRESS)
 413                                 start = ISA_END_ADDRESS;
 414                 } else
 415                         start = ISA_END_ADDRESS;
 416                 new_mapped_ram_size = init_range_memory_mapping(start,
 417                                                         last_start);
 418                 last_start = start;
 419                 min_pfn_mapped = last_start >> PAGE_SHIFT;
 420                 /* only increase step_size after big range get mapped */
 421                 if (new_mapped_ram_size > mapped_ram_size)
 422                         step_size <<= STEP_SIZE_SHIFT;
 423                 mapped_ram_size += new_mapped_ram_size;
 424         }
 425
 426         if (real_end < end)
 427                 init_range_memory_mapping(real_end, end);
 428
 429 #ifdef CONFIG_X86_64
 430         if (max_pfn > max_low_pfn) {
 431                 /* can we preseve max_low_pfn ?*/
 432                 max_low_pfn = max_pfn;
 433         }
 434 #endif
 435         early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
 436 }
 437
 438 /*
 439  * devmem_is_allowed() checks to see if /dev/mem access to a certain address
 440  * is valid. The argument is a physical page number.
 441  *
 442  *
 443  * On x86, access has to be given to the first megabyte of ram because that area
 444  * contains bios code and data regions used by X and dosemu and similar apps.
 445  * Access has to be given to non-kernel-ram areas as well, these contain the PCI
 446  * mmio resources as well as potential bios/acpi data regions.
 447  */
 448 int devmem_is_allowed(unsigned long pagenr)
 449 {
 450         if (pagenr < 256)
 451                 return 1;
 452         if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
 453                 return 0;
 454         if (!page_is_ram(pagenr))
 455                 return 1;
 456         return 0;
 457 }
 458
 459 void free_init_pages(char *what, unsigned long begin, unsigned long end)
 460 {
 461         unsigned long addr;
 462         unsigned long begin_aligned, end_aligned;
 463
 464         /* Make sure boundaries are page aligned */
 465         begin_aligned = PAGE_ALIGN(begin);
 466         end_aligned   = end & PAGE_MASK;
 467
 468         if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
 469                 begin = begin_aligned;
 470                 end   = end_aligned;
 471         }
 472
 473         if (begin >= end)
 474                 return;
 475
 476         addr = begin;
 477
 478         /*
 479          * If debugging page accesses then do not free this memory but
 480          * mark them not present - any buggy init-section access will
 481          * create a kernel page fault:
 482          */
 483 #ifdef CONFIG_DEBUG_PAGEALLOC
 484         printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n",
 485                 begin, end - 1);
 486         set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
 487 #else
 488         /*
 489          * We just marked the kernel text read only above, now that
 490          * we are going to free part of that, we need to make that
 491          * writeable and non-executable first.
 492          */
 493         set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
 494         set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
 495
 496         printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
 497
 498         for (; addr < end; addr += PAGE_SIZE) {
 499                 ClearPageReserved(virt_to_page(addr));
 500                 init_page_count(virt_to_page(addr));
 501                 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
 502                 free_page(addr);
 503                 totalram_pages++;
 504         }
 505 #endif
 506 }
 507
 508 void free_initmem(void)
 509 {
 510         free_init_pages("unused kernel memory",
 511                         (unsigned long)(&__init_begin),
 512                         (unsigned long)(&__init_end));
 513 }
 514
 515 #ifdef CONFIG_BLK_DEV_INITRD
 516 void __init free_initrd_mem(unsigned long start, unsigned long end)
 517 {
 518         /*
 519          * end could be not aligned, and We can not align that,
 520          * decompresser could be confused by aligned initrd_end
 521          * We already reserve the end partial page before in
 522          *   - i386_start_kernel()
 523          *   - x86_64_start_kernel()
 524          *   - relocate_initrd()
 525          * So here We can do PAGE_ALIGN() safely to get partial page to be freed
 526          */
 527         free_init_pages("initrd memory", start, PAGE_ALIGN(end));
 528 }
 529 #endif
 530
 531 void __init zone_sizes_init(void)
 532 {
 533         unsigned long max_zone_pfns[MAX_NR_ZONES];
 534
 535         memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
 536
 537 #ifdef CONFIG_ZONE_DMA
 538         max_zone_pfns[ZONE_DMA]         = MAX_DMA_PFN;
 539 #endif
 540 #ifdef CONFIG_ZONE_DMA32
 541         max_zone_pfns[ZONE_DMA32]       = MAX_DMA32_PFN;
 542 #endif
 543         max_zone_pfns[ZONE_NORMAL]      = max_low_pfn;
 544 #ifdef CONFIG_HIGHMEM
 545         max_zone_pfns[ZONE_HIGHMEM]     = max_pfn;
 546 #endif
 547
 548         free_area_init_nodes(max_zone_pfns);
 549 }
 550