1 #include <linux/module.h>
2 #include <linux/sched.h>
3 #include <linux/mutex.h>
4 #include <linux/list.h>
5 #include <linux/stringify.h>
6 #include <linux/kprobes.h>
8 #include <linux/vmalloc.h>
9 #include <linux/memory.h>
10 #include <linux/stop_machine.h>
11 #include <linux/slab.h>
12 #include <asm/alternative.h>
13 #include <asm/sections.h>
14 #include <asm/pgtable.h>
17 #include <asm/vsyscall.h>
18 #include <asm/cacheflush.h>
19 #include <asm/tlbflush.h>
21 #include <asm/fixmap.h>
23 #define MAX_PATCH_LEN (255-1)
25 #ifdef CONFIG_HOTPLUG_CPU
26 static int smp_alt_once;
28 static int __init bootonly(char *str)
33 __setup("smp-alt-boot", bootonly);
35 #define smp_alt_once 1
38 static int __initdata_or_module debug_alternative;
40 static int __init debug_alt(char *str)
42 debug_alternative = 1;
45 __setup("debug-alternative", debug_alt);
47 static int noreplace_smp;
49 static int __init setup_noreplace_smp(char *str)
54 __setup("noreplace-smp", setup_noreplace_smp);
56 #ifdef CONFIG_PARAVIRT
57 static int __initdata_or_module noreplace_paravirt = 0;
59 static int __init setup_noreplace_paravirt(char *str)
61 noreplace_paravirt = 1;
64 __setup("noreplace-paravirt", setup_noreplace_paravirt);
67 #define DPRINTK(fmt, args...) if (debug_alternative) \
68 printk(KERN_DEBUG fmt, args)
70 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
71 /* Use inline assembly to define this because the nops are defined
72 as inline assembly strings in the include files and we cannot
73 get them easily into strings. */
74 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: "
75 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
76 GENERIC_NOP7 GENERIC_NOP8
78 extern const unsigned char intelnops[];
79 static const unsigned char *const __initconst_or_module
80 intel_nops[ASM_NOP_MAX+1] = {
85 intelnops + 1 + 2 + 3,
86 intelnops + 1 + 2 + 3 + 4,
87 intelnops + 1 + 2 + 3 + 4 + 5,
88 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
89 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
94 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: "
95 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
98 extern const unsigned char k8nops[];
99 static const unsigned char *const __initconst_or_module
100 k8_nops[ASM_NOP_MAX+1] = {
106 k8nops + 1 + 2 + 3 + 4,
107 k8nops + 1 + 2 + 3 + 4 + 5,
108 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
109 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
113 #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
114 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: "
115 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
118 extern const unsigned char k7nops[];
119 static const unsigned char *const __initconst_or_module
120 k7_nops[ASM_NOP_MAX+1] = {
126 k7nops + 1 + 2 + 3 + 4,
127 k7nops + 1 + 2 + 3 + 4 + 5,
128 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
129 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
134 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: "
135 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
138 extern const unsigned char p6nops[];
139 static const unsigned char *const __initconst_or_module
140 p6_nops[ASM_NOP_MAX+1] = {
146 p6nops + 1 + 2 + 3 + 4,
147 p6nops + 1 + 2 + 3 + 4 + 5,
148 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
149 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
155 extern char __vsyscall_0;
156 static const unsigned char *const *__init_or_module find_nop_table(void)
158 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
159 boot_cpu_has(X86_FEATURE_NOPL))
165 #else /* CONFIG_X86_64 */
167 static const unsigned char *const *__init_or_module find_nop_table(void)
169 if (boot_cpu_has(X86_FEATURE_K8))
171 else if (boot_cpu_has(X86_FEATURE_K7))
173 else if (boot_cpu_has(X86_FEATURE_NOPL))
179 #endif /* CONFIG_X86_64 */
181 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
182 static void __init_or_module add_nops(void *insns, unsigned int len)
184 const unsigned char *const *noptable = find_nop_table();
187 unsigned int noplen = len;
188 if (noplen > ASM_NOP_MAX)
189 noplen = ASM_NOP_MAX;
190 memcpy(insns, noptable[noplen], noplen);
196 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
197 extern s32 __smp_locks[], __smp_locks_end[];
198 static void *text_poke_early(void *addr, const void *opcode, size_t len);
200 /* Replace instructions with better alternatives for this CPU type.
201 This runs before SMP is initialized to avoid SMP problems with
202 self modifying code. This implies that assymetric systems where
203 APs have less capabilities than the boot processor are not handled.
204 Tough. Make sure you disable such features by hand. */
206 void __init_or_module apply_alternatives(struct alt_instr *start,
207 struct alt_instr *end)
210 u8 insnbuf[MAX_PATCH_LEN];
212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
213 for (a = start; a < end; a++) {
214 u8 *instr = a->instr;
215 BUG_ON(a->replacementlen > a->instrlen);
216 BUG_ON(a->instrlen > sizeof(insnbuf));
217 if (!boot_cpu_has(a->cpuid))
220 /* vsyscall code is not mapped yet. resolve it manually. */
221 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
222 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
223 DPRINTK("%s: vsyscall fixup: %p => %p\n",
224 __func__, a->instr, instr);
227 memcpy(insnbuf, a->replacement, a->replacementlen);
228 if (*insnbuf == 0xe8 && a->replacementlen == 5)
229 *(s32 *)(insnbuf + 1) += a->replacement - a->instr;
230 add_nops(insnbuf + a->replacementlen,
231 a->instrlen - a->replacementlen);
232 text_poke_early(instr, insnbuf, a->instrlen);
238 static void alternatives_smp_lock(const s32 *start, const s32 *end,
239 u8 *text, u8 *text_end)
243 mutex_lock(&text_mutex);
244 for (poff = start; poff < end; poff++) {
245 u8 *ptr = (u8 *)poff + *poff;
247 if (!*poff || ptr < text || ptr >= text_end)
249 /* turn DS segment override prefix into lock prefix */
250 text_poke(ptr, ((unsigned char []){0xf0}), 1);
252 mutex_unlock(&text_mutex);
255 static void alternatives_smp_unlock(const s32 *start, const s32 *end,
256 u8 *text, u8 *text_end)
263 mutex_lock(&text_mutex);
264 for (poff = start; poff < end; poff++) {
265 u8 *ptr = (u8 *)poff + *poff;
267 if (!*poff || ptr < text || ptr >= text_end)
269 /* turn lock prefix into DS segment override prefix */
270 text_poke(ptr, ((unsigned char []){0x3E}), 1);
272 mutex_unlock(&text_mutex);
275 struct smp_alt_module {
276 /* what is this ??? */
280 /* ptrs to lock prefixes */
282 const s32 *locks_end;
284 /* .text segment, needed to avoid patching init code ;) */
288 struct list_head next;
290 static LIST_HEAD(smp_alt_modules);
291 static DEFINE_MUTEX(smp_alt);
292 static int smp_mode = 1; /* protected by smp_alt */
294 void __init_or_module alternatives_smp_module_add(struct module *mod,
296 void *locks, void *locks_end,
297 void *text, void *text_end)
299 struct smp_alt_module *smp;
305 if (boot_cpu_has(X86_FEATURE_UP))
306 alternatives_smp_unlock(locks, locks_end,
311 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
313 return; /* we'll run the (safe but slow) SMP code then ... */
318 smp->locks_end = locks_end;
320 smp->text_end = text_end;
321 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
322 __func__, smp->locks, smp->locks_end,
323 smp->text, smp->text_end, smp->name);
325 mutex_lock(&smp_alt);
326 list_add_tail(&smp->next, &smp_alt_modules);
327 if (boot_cpu_has(X86_FEATURE_UP))
328 alternatives_smp_unlock(smp->locks, smp->locks_end,
329 smp->text, smp->text_end);
330 mutex_unlock(&smp_alt);
333 void __init_or_module alternatives_smp_module_del(struct module *mod)
335 struct smp_alt_module *item;
337 if (smp_alt_once || noreplace_smp)
340 mutex_lock(&smp_alt);
341 list_for_each_entry(item, &smp_alt_modules, next) {
342 if (mod != item->mod)
344 list_del(&item->next);
345 mutex_unlock(&smp_alt);
346 DPRINTK("%s: %s\n", __func__, item->name);
350 mutex_unlock(&smp_alt);
353 void alternatives_smp_switch(int smp)
355 struct smp_alt_module *mod;
357 #ifdef CONFIG_LOCKDEP
359 * Older binutils section handling bug prevented
360 * alternatives-replacement from working reliably.
362 * If this still occurs then you should see a hang
363 * or crash shortly after this line:
365 printk("lockdep: fixing up alternatives.\n");
368 if (noreplace_smp || smp_alt_once)
370 BUG_ON(!smp && (num_online_cpus() > 1));
372 mutex_lock(&smp_alt);
375 * Avoid unnecessary switches because it forces JIT based VMs to
376 * throw away all cached translations, which can be quite costly.
378 if (smp == smp_mode) {
381 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
382 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
383 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
384 list_for_each_entry(mod, &smp_alt_modules, next)
385 alternatives_smp_lock(mod->locks, mod->locks_end,
386 mod->text, mod->text_end);
388 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
389 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
390 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
391 list_for_each_entry(mod, &smp_alt_modules, next)
392 alternatives_smp_unlock(mod->locks, mod->locks_end,
393 mod->text, mod->text_end);
396 mutex_unlock(&smp_alt);
399 /* Return 1 if the address range is reserved for smp-alternatives */
400 int alternatives_text_reserved(void *start, void *end)
402 struct smp_alt_module *mod;
404 u8 *text_start = start;
407 list_for_each_entry(mod, &smp_alt_modules, next) {
408 if (mod->text > text_end || mod->text_end < text_start)
410 for (poff = mod->locks; poff < mod->locks_end; poff++) {
411 const u8 *ptr = (const u8 *)poff + *poff;
413 if (text_start <= ptr && text_end > ptr)
422 #ifdef CONFIG_PARAVIRT
423 void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
424 struct paravirt_patch_site *end)
426 struct paravirt_patch_site *p;
427 char insnbuf[MAX_PATCH_LEN];
429 if (noreplace_paravirt)
432 for (p = start; p < end; p++) {
435 BUG_ON(p->len > MAX_PATCH_LEN);
436 /* prep the buffer with the original instructions */
437 memcpy(insnbuf, p->instr, p->len);
438 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
439 (unsigned long)p->instr, p->len);
441 BUG_ON(used > p->len);
443 /* Pad the rest with nops */
444 add_nops(insnbuf + used, p->len - used);
445 text_poke_early(p->instr, insnbuf, p->len);
448 extern struct paravirt_patch_site __start_parainstructions[],
449 __stop_parainstructions[];
450 #endif /* CONFIG_PARAVIRT */
452 void __init alternative_instructions(void)
454 /* The patching is not fully atomic, so try to avoid local interruptions
455 that might execute the to be patched code.
456 Other CPUs are not running. */
460 * Don't stop machine check exceptions while patching.
461 * MCEs only happen when something got corrupted and in this
462 * case we must do something about the corruption.
463 * Ignoring it is worse than a unlikely patching race.
464 * Also machine checks tend to be broadcast and if one CPU
465 * goes into machine check the others follow quickly, so we don't
466 * expect a machine check to cause undue problems during to code
470 apply_alternatives(__alt_instructions, __alt_instructions_end);
472 /* switch to patch-once-at-boottime-only mode and free the
473 * tables in case we know the number of CPUs will never ever
475 #ifdef CONFIG_HOTPLUG_CPU
476 if (num_possible_cpus() < 2)
482 if (1 == num_possible_cpus()) {
483 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
484 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
485 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
487 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
491 alternatives_smp_module_add(NULL, "core kernel",
492 __smp_locks, __smp_locks_end,
495 /* Only switch to UP mode if we don't immediately boot others */
496 if (num_present_cpus() == 1 || setup_max_cpus <= 1)
497 alternatives_smp_switch(0);
500 apply_paravirt(__parainstructions, __parainstructions_end);
503 free_init_pages("SMP alternatives",
504 (unsigned long)__smp_locks,
505 (unsigned long)__smp_locks_end);
511 * text_poke_early - Update instructions on a live kernel at boot time
512 * @addr: address to modify
513 * @opcode: source of the copy
514 * @len: length to copy
516 * When you use this code to patch more than one byte of an instruction
517 * you need to make sure that other CPUs cannot execute this code in parallel.
518 * Also no thread must be currently preempted in the middle of these
519 * instructions. And on the local CPU you need to be protected again NMI or MCE
520 * handlers seeing an inconsistent instruction while you patch.
522 static void *__init_or_module text_poke_early(void *addr, const void *opcode,
526 local_irq_save(flags);
527 memcpy(addr, opcode, len);
529 local_irq_restore(flags);
530 /* Could also do a CLFLUSH here to speed up CPU recovery; but
531 that causes hangs on some VIA CPUs. */
536 * text_poke - Update instructions on a live kernel
537 * @addr: address to modify
538 * @opcode: source of the copy
539 * @len: length to copy
541 * Only atomic text poke/set should be allowed when not doing early patching.
542 * It means the size must be writable atomically and the address must be aligned
543 * in a way that permits an atomic write. It also makes sure we fit on a single
546 * Note: Must be called under text_mutex.
548 void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
552 struct page *pages[2];
555 if (!core_kernel_text((unsigned long)addr)) {
556 pages[0] = vmalloc_to_page(addr);
557 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
559 pages[0] = virt_to_page(addr);
560 WARN_ON(!PageReserved(pages[0]));
561 pages[1] = virt_to_page(addr + PAGE_SIZE);
564 local_irq_save(flags);
565 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
567 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
568 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
569 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
570 clear_fixmap(FIX_TEXT_POKE0);
572 clear_fixmap(FIX_TEXT_POKE1);
575 /* Could also do a CLFLUSH here to speed up CPU recovery; but
576 that causes hangs on some VIA CPUs. */
577 for (i = 0; i < len; i++)
578 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
579 local_irq_restore(flags);
584 * Cross-modifying kernel text with stop_machine().
585 * This code originally comes from immediate value.
587 static atomic_t stop_machine_first;
588 static int wrote_text;
590 struct text_poke_params {
596 static int __kprobes stop_machine_text_poke(void *data)
598 struct text_poke_params *tpp = data;
600 if (atomic_dec_and_test(&stop_machine_first)) {
601 text_poke(tpp->addr, tpp->opcode, tpp->len);
602 smp_wmb(); /* Make sure other cpus see that this has run */
607 smp_mb(); /* Load wrote_text before following execution */
610 flush_icache_range((unsigned long)tpp->addr,
611 (unsigned long)tpp->addr + tpp->len);
616 * text_poke_smp - Update instructions on a live kernel on SMP
617 * @addr: address to modify
618 * @opcode: source of the copy
619 * @len: length to copy
621 * Modify multi-byte instruction by using stop_machine() on SMP. This allows
622 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
623 * should be allowed, since stop_machine() does _not_ protect code against
626 * Note: Must be called under get_online_cpus() and text_mutex.
628 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
630 struct text_poke_params tpp;
635 atomic_set(&stop_machine_first, 1);
637 stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);