2 * Memory subsystem support
4 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
5 * Dave Hansen <haveblue@us.ibm.com>
7 * This file provides the necessary infrastructure to represent
8 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
9 * All arch-independent code that assumes MEMORY_HOTPLUG requires
10 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/topology.h>
16 #include <linux/capability.h>
17 #include <linux/device.h>
18 #include <linux/memory.h>
19 #include <linux/memory_hotplug.h>
21 #include <linux/mutex.h>
22 #include <linux/stat.h>
23 #include <linux/slab.h>
25 #include <linux/atomic.h>
26 #include <asm/uaccess.h>
28 static DEFINE_MUTEX(mem_sysfs_mutex);
30 #define MEMORY_CLASS_NAME "memory"
32 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
34 static int sections_per_block;
36 static inline int base_memory_block_id(int section_nr)
38 return section_nr / sections_per_block;
41 static int memory_subsys_online(struct device *dev);
42 static int memory_subsys_offline(struct device *dev);
44 static struct bus_type memory_subsys = {
45 .name = MEMORY_CLASS_NAME,
46 .dev_name = MEMORY_CLASS_NAME,
47 .online = memory_subsys_online,
48 .offline = memory_subsys_offline,
51 static BLOCKING_NOTIFIER_HEAD(memory_chain);
53 int register_memory_notifier(struct notifier_block *nb)
55 return blocking_notifier_chain_register(&memory_chain, nb);
57 EXPORT_SYMBOL(register_memory_notifier);
59 void unregister_memory_notifier(struct notifier_block *nb)
61 blocking_notifier_chain_unregister(&memory_chain, nb);
63 EXPORT_SYMBOL(unregister_memory_notifier);
65 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain);
67 int register_memory_isolate_notifier(struct notifier_block *nb)
69 return atomic_notifier_chain_register(&memory_isolate_chain, nb);
71 EXPORT_SYMBOL(register_memory_isolate_notifier);
73 void unregister_memory_isolate_notifier(struct notifier_block *nb)
75 atomic_notifier_chain_unregister(&memory_isolate_chain, nb);
77 EXPORT_SYMBOL(unregister_memory_isolate_notifier);
79 static void memory_block_release(struct device *dev)
81 struct memory_block *mem = to_memory_block(dev);
86 unsigned long __weak memory_block_size_bytes(void)
88 return MIN_MEMORY_BLOCK_SIZE;
91 static unsigned long get_memory_block_size(void)
93 unsigned long block_sz;
95 block_sz = memory_block_size_bytes();
97 /* Validate blk_sz is a power of 2 and not less than section size */
98 if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) {
100 block_sz = MIN_MEMORY_BLOCK_SIZE;
107 * use this as the physical section index that this memsection
111 static ssize_t show_mem_start_phys_index(struct device *dev,
112 struct device_attribute *attr, char *buf)
114 struct memory_block *mem = to_memory_block(dev);
115 unsigned long phys_index;
117 phys_index = mem->start_section_nr / sections_per_block;
118 return sprintf(buf, "%08lx\n", phys_index);
121 static ssize_t show_mem_end_phys_index(struct device *dev,
122 struct device_attribute *attr, char *buf)
124 struct memory_block *mem = to_memory_block(dev);
125 unsigned long phys_index;
127 phys_index = mem->end_section_nr / sections_per_block;
128 return sprintf(buf, "%08lx\n", phys_index);
132 * Show whether the section of memory is likely to be hot-removable
134 static ssize_t show_mem_removable(struct device *dev,
135 struct device_attribute *attr, char *buf)
137 unsigned long i, pfn;
139 struct memory_block *mem = to_memory_block(dev);
141 for (i = 0; i < sections_per_block; i++) {
142 pfn = section_nr_to_pfn(mem->start_section_nr + i);
143 ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION);
146 return sprintf(buf, "%d\n", ret);
150 * online, offline, going offline, etc.
152 static ssize_t show_mem_state(struct device *dev,
153 struct device_attribute *attr, char *buf)
155 struct memory_block *mem = to_memory_block(dev);
159 * We can probably put these states in a nice little array
160 * so that they're not open-coded
162 switch (mem->state) {
164 len = sprintf(buf, "online\n");
167 len = sprintf(buf, "offline\n");
169 case MEM_GOING_OFFLINE:
170 len = sprintf(buf, "going-offline\n");
173 len = sprintf(buf, "ERROR-UNKNOWN-%ld\n",
182 int memory_notify(unsigned long val, void *v)
184 return blocking_notifier_call_chain(&memory_chain, val, v);
187 int memory_isolate_notify(unsigned long val, void *v)
189 return atomic_notifier_call_chain(&memory_isolate_chain, val, v);
193 * The probe routines leave the pages reserved, just as the bootmem code does.
194 * Make sure they're still that way.
196 static bool pages_correctly_reserved(unsigned long start_pfn)
200 unsigned long pfn = start_pfn;
203 * memmap between sections is not contiguous except with
204 * SPARSEMEM_VMEMMAP. We lookup the page once per section
205 * and assume memmap is contiguous within each section
207 for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) {
208 if (WARN_ON_ONCE(!pfn_valid(pfn)))
210 page = pfn_to_page(pfn);
212 for (j = 0; j < PAGES_PER_SECTION; j++) {
213 if (PageReserved(page + j))
216 printk(KERN_WARNING "section number %ld page number %d "
217 "not reserved, was it already online?\n",
218 pfn_to_section_nr(pfn), j);
228 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
229 * OK to have direct references to sparsemem variables in here.
232 memory_block_action(unsigned long phys_index, unsigned long action, int online_type)
234 unsigned long start_pfn;
235 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
236 struct page *first_page;
239 first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT);
240 start_pfn = page_to_pfn(first_page);
244 if (!pages_correctly_reserved(start_pfn))
247 ret = online_pages(start_pfn, nr_pages, online_type);
250 ret = offline_pages(start_pfn, nr_pages);
253 WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
254 "%ld\n", __func__, phys_index, action, action);
261 static int memory_block_change_state(struct memory_block *mem,
262 unsigned long to_state, unsigned long from_state_req)
266 if (mem->state != from_state_req)
269 if (to_state == MEM_OFFLINE)
270 mem->state = MEM_GOING_OFFLINE;
272 ret = memory_block_action(mem->start_section_nr, to_state,
275 mem->state = ret ? from_state_req : to_state;
280 /* The device lock serializes operations on memory_subsys_[online|offline] */
281 static int memory_subsys_online(struct device *dev)
283 struct memory_block *mem = to_memory_block(dev);
286 if (mem->state == MEM_ONLINE)
290 * If we are called from store_mem_state(), online_type will be
291 * set >= 0 Otherwise we were called from the device online
292 * attribute and need to set the online_type.
294 if (mem->online_type < 0)
295 mem->online_type = ONLINE_KEEP;
297 ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
299 /* clear online_type */
300 mem->online_type = -1;
305 static int memory_subsys_offline(struct device *dev)
307 struct memory_block *mem = to_memory_block(dev);
309 if (mem->state == MEM_OFFLINE)
312 return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);
316 store_mem_state(struct device *dev,
317 struct device_attribute *attr, const char *buf, size_t count)
319 struct memory_block *mem = to_memory_block(dev);
320 int ret, online_type;
322 lock_device_hotplug();
324 if (!strncmp(buf, "online_kernel", min_t(int, count, 13)))
325 online_type = ONLINE_KERNEL;
326 else if (!strncmp(buf, "online_movable", min_t(int, count, 14)))
327 online_type = ONLINE_MOVABLE;
328 else if (!strncmp(buf, "online", min_t(int, count, 6)))
329 online_type = ONLINE_KEEP;
330 else if (!strncmp(buf, "offline", min_t(int, count, 7)))
335 switch (online_type) {
340 * mem->online_type is not protected so there can be a
341 * race here. However, when racing online, the first
342 * will succeed and the second will just return as the
343 * block will already be online. The online type
344 * could be either one, but that is expected.
346 mem->online_type = online_type;
347 ret = device_online(&mem->dev);
350 ret = device_offline(&mem->dev);
353 ret = -EINVAL; /* should never happen */
356 unlock_device_hotplug();
364 * phys_device is a bad name for this. What I really want
365 * is a way to differentiate between memory ranges that
366 * are part of physical devices that constitute
367 * a complete removable unit or fru.
368 * i.e. do these ranges belong to the same physical device,
369 * s.t. if I offline all of these sections I can then
370 * remove the physical device?
372 static ssize_t show_phys_device(struct device *dev,
373 struct device_attribute *attr, char *buf)
375 struct memory_block *mem = to_memory_block(dev);
376 return sprintf(buf, "%d\n", mem->phys_device);
379 static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL);
380 static DEVICE_ATTR(end_phys_index, 0444, show_mem_end_phys_index, NULL);
381 static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state);
382 static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL);
383 static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL);
386 * Block size attribute stuff
389 print_block_size(struct device *dev, struct device_attribute *attr,
392 return sprintf(buf, "%lx\n", get_memory_block_size());
395 static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL);
398 * Some architectures will have custom drivers to do this, and
399 * will not need to do it from userspace. The fake hot-add code
400 * as well as ppc64 will do all of their discovery in userspace
401 * and will require this interface.
403 #ifdef CONFIG_ARCH_MEMORY_PROBE
405 memory_probe_store(struct device *dev, struct device_attribute *attr,
406 const char *buf, size_t count)
411 unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
413 phys_addr = simple_strtoull(buf, NULL, 0);
415 if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
418 for (i = 0; i < sections_per_block; i++) {
419 nid = memory_add_physaddr_to_nid(phys_addr);
420 ret = add_memory(nid, phys_addr,
421 PAGES_PER_SECTION << PAGE_SHIFT);
425 phys_addr += MIN_MEMORY_BLOCK_SIZE;
433 static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store);
436 #ifdef CONFIG_MEMORY_FAILURE
438 * Support for offlining pages of memory
441 /* Soft offline a page */
443 store_soft_offline_page(struct device *dev,
444 struct device_attribute *attr,
445 const char *buf, size_t count)
449 if (!capable(CAP_SYS_ADMIN))
451 if (kstrtoull(buf, 0, &pfn) < 0)
456 ret = soft_offline_page(pfn_to_page(pfn), 0);
457 return ret == 0 ? count : ret;
460 /* Forcibly offline a page, including killing processes. */
462 store_hard_offline_page(struct device *dev,
463 struct device_attribute *attr,
464 const char *buf, size_t count)
468 if (!capable(CAP_SYS_ADMIN))
470 if (kstrtoull(buf, 0, &pfn) < 0)
473 ret = memory_failure(pfn, 0, 0);
474 return ret ? ret : count;
477 static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page);
478 static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page);
482 * Note that phys_device is optional. It is here to allow for
483 * differentiation between which *physical* devices each
484 * section belongs to...
486 int __weak arch_get_memory_phys_device(unsigned long start_pfn)
492 * A reference for the returned object is held and the reference for the
493 * hinted object is released.
495 struct memory_block *find_memory_block_hinted(struct mem_section *section,
496 struct memory_block *hint)
498 int block_id = base_memory_block_id(__section_nr(section));
499 struct device *hintdev = hint ? &hint->dev : NULL;
502 dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
504 put_device(&hint->dev);
507 return to_memory_block(dev);
511 * For now, we have a linear search to go find the appropriate
512 * memory_block corresponding to a particular phys_index. If
513 * this gets to be a real problem, we can always use a radix
514 * tree or something here.
516 * This could be made generic for all device subsystems.
518 struct memory_block *find_memory_block(struct mem_section *section)
520 return find_memory_block_hinted(section, NULL);
523 static struct attribute *memory_memblk_attrs[] = {
524 &dev_attr_phys_index.attr,
525 &dev_attr_end_phys_index.attr,
526 &dev_attr_state.attr,
527 &dev_attr_phys_device.attr,
528 &dev_attr_removable.attr,
532 static struct attribute_group memory_memblk_attr_group = {
533 .attrs = memory_memblk_attrs,
536 static const struct attribute_group *memory_memblk_attr_groups[] = {
537 &memory_memblk_attr_group,
542 * register_memory - Setup a sysfs device for a memory block
545 int register_memory(struct memory_block *memory)
547 memory->dev.bus = &memory_subsys;
548 memory->dev.id = memory->start_section_nr / sections_per_block;
549 memory->dev.release = memory_block_release;
550 memory->dev.groups = memory_memblk_attr_groups;
551 memory->dev.offline = memory->state == MEM_OFFLINE;
553 return device_register(&memory->dev);
556 static int init_memory_block(struct memory_block **memory,
557 struct mem_section *section, unsigned long state)
559 struct memory_block *mem;
560 unsigned long start_pfn;
564 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
568 scn_nr = __section_nr(section);
569 mem->start_section_nr =
570 base_memory_block_id(scn_nr) * sections_per_block;
571 mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
573 mem->section_count++;
574 start_pfn = section_nr_to_pfn(mem->start_section_nr);
575 mem->phys_device = arch_get_memory_phys_device(start_pfn);
577 ret = register_memory(mem);
583 static int add_memory_block(int base_section_nr)
585 struct memory_block *mem;
586 int i, ret, section_count = 0, section_nr;
588 for (i = base_section_nr;
589 (i < base_section_nr + sections_per_block) && i < NR_MEM_SECTIONS;
591 if (!present_section_nr(i))
593 if (section_count == 0)
598 if (section_count == 0)
600 ret = init_memory_block(&mem, __nr_to_section(section_nr), MEM_ONLINE);
603 mem->section_count = section_count;
609 * need an interface for the VM to add new memory regions,
610 * but without onlining it.
612 int register_new_memory(int nid, struct mem_section *section)
615 struct memory_block *mem;
617 mutex_lock(&mem_sysfs_mutex);
619 mem = find_memory_block(section);
621 mem->section_count++;
622 put_device(&mem->dev);
624 ret = init_memory_block(&mem, section, MEM_OFFLINE);
629 if (mem->section_count == sections_per_block)
630 ret = register_mem_sect_under_node(mem, nid);
632 mutex_unlock(&mem_sysfs_mutex);
636 #ifdef CONFIG_MEMORY_HOTREMOVE
638 unregister_memory(struct memory_block *memory)
640 BUG_ON(memory->dev.bus != &memory_subsys);
642 /* drop the ref. we got in remove_memory_block() */
643 put_device(&memory->dev);
644 device_unregister(&memory->dev);
647 static int remove_memory_block(unsigned long node_id,
648 struct mem_section *section, int phys_device)
650 struct memory_block *mem;
652 mutex_lock(&mem_sysfs_mutex);
653 mem = find_memory_block(section);
654 unregister_mem_sect_under_nodes(mem, __section_nr(section));
656 mem->section_count--;
657 if (mem->section_count == 0)
658 unregister_memory(mem);
660 put_device(&mem->dev);
662 mutex_unlock(&mem_sysfs_mutex);
666 int unregister_memory_section(struct mem_section *section)
668 if (!present_section(section))
671 return remove_memory_block(0, section, 0);
673 #endif /* CONFIG_MEMORY_HOTREMOVE */
675 /* return true if the memory block is offlined, otherwise, return false */
676 bool is_memblock_offlined(struct memory_block *mem)
678 return mem->state == MEM_OFFLINE;
681 static struct attribute *memory_root_attrs[] = {
682 #ifdef CONFIG_ARCH_MEMORY_PROBE
683 &dev_attr_probe.attr,
686 #ifdef CONFIG_MEMORY_FAILURE
687 &dev_attr_soft_offline_page.attr,
688 &dev_attr_hard_offline_page.attr,
691 &dev_attr_block_size_bytes.attr,
695 static struct attribute_group memory_root_attr_group = {
696 .attrs = memory_root_attrs,
699 static const struct attribute_group *memory_root_attr_groups[] = {
700 &memory_root_attr_group,
705 * Initialize the sysfs support for memory devices...
707 int __init memory_dev_init(void)
712 unsigned long block_sz;
714 ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
718 block_sz = get_memory_block_size();
719 sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
722 * Create entries for memory sections that were found
723 * during boot and have been initialized
725 mutex_lock(&mem_sysfs_mutex);
726 for (i = 0; i < NR_MEM_SECTIONS; i += sections_per_block) {
727 err = add_memory_block(i);
731 mutex_unlock(&mem_sysfs_mutex);
735 printk(KERN_ERR "%s() failed: %d\n", __func__, ret);