Merge branch 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux...

* 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen:
  jump-label: initialize jump-label subsystem much earlier
  x86/jump_label: add arch_jump_label_transform_static()
  s390/jump-label: add arch_jump_label_transform_static()
  jump_label: add arch_jump_label_transform_static() to optimise non-live code updates
  sparc/jump_label: drop arch_jump_label_text_poke_early()
  x86/jump_label: drop arch_jump_label_text_poke_early()
  jump_label: if a key has already been initialized, don't nop it out
  stop_machine: make stop_machine safe and efficient to call early
  jump_label: use proper atomic_t initializer

Conflicts:
 - arch/x86/kernel/jump_label.c
	Added __init_or_module to arch_jump_label_text_poke_early vs
	removal of that function entirely
 - kernel/stop_machine.c
	same patch ("stop_machine: make stop_machine safe and efficient
	to call early") merged twice, with whitespace fix in one version

diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c
index 44cc06bedf77b0985fc93a67582f9ece7b016bf5..b987ab2c1541afb1866e11949c156e6b62c9b67c 100644 (file)
--- a/arch/s390/kernel/jump_label.c
+++ b/arch/s390/kernel/jump_label.c
@@ -18,26 +18,15 @@ struct insn {
 } __packed;
 
 struct insn_args {
-       unsigned long *target;
-       struct insn *insn;
-       ssize_t size;
+       struct jump_entry *entry;
+       enum jump_label_type type;
 };
 
-static int __arch_jump_label_transform(void *data)
+static void __jump_label_transform(struct jump_entry *entry,
+                                  enum jump_label_type type)
 {
-       struct insn_args *args = data;
-       int rc;
-
-       rc = probe_kernel_write(args->target, args->insn, args->size);
-       WARN_ON_ONCE(rc < 0);
-       return 0;
-}
-
-void arch_jump_label_transform(struct jump_entry *entry,
-                              enum jump_label_type type)
-{
-       struct insn_args args;
        struct insn insn;
+       int rc;
 
        if (type == JUMP_LABEL_ENABLE) {
                /* brcl 15,offset */
@@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry,
                insn.offset = 0;
        }
 
-       args.target = (void *) entry->code;
-       args.insn = &insn;
-       args.size = JUMP_LABEL_NOP_SIZE;
+       rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE);
+       WARN_ON_ONCE(rc < 0);
+}
 
-       stop_machine(__arch_jump_label_transform, &args, NULL);
+static int __sm_arch_jump_label_transform(void *data)
+{
+       struct insn_args *args = data;
+
+       __jump_label_transform(args->entry, args->type);
+       return 0;
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       struct insn_args args;
+
+       args.entry = entry;
+       args.type = type;
+
+       stop_machine(__sm_arch_jump_label_transform, &args, NULL);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       __jump_label_transform(entry, type);
 }
 
 #endif

diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
index ea2dafc93d780fd0b89e390af3d4aca7c106fdf7..971fd435a281767d80bd75db250afad41b847745 100644 (file)
--- a/arch/sparc/kernel/jump_label.c
+++ b/arch/sparc/kernel/jump_label.c
@@ -36,12 +36,4 @@ void arch_jump_label_transform(struct jump_entry *entry,
        put_online_cpus();
 }
 
-void arch_jump_label_text_poke_early(jump_label_t addr)
-{
-       u32 *insn_p = (u32 *) (unsigned long) addr;
-
-       *insn_p = 0x01000000;
-       flushi(insn_p);
-}
-
 #endif

diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index cacdd46d184d4c0253540db3d3a2f9ed96047253..ea9d5f2f13efec411a79a0b43f003b29c06654d3 100644 (file)
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -24,8 +24,9 @@ union jump_code_union {
        } __attribute__((packed));
 };
 
-void arch_jump_label_transform(struct jump_entry *entry,
-                              enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *entry,
+                                  enum jump_label_type type,
+                                  void *(*poker)(void *, const void *, size_t))
 {
        union jump_code_union code;
 
@@ -35,17 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry,
                                (entry->code + JUMP_LABEL_NOP_SIZE);
        } else
                memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+
+       (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
        get_online_cpus();
        mutex_lock(&text_mutex);
-       text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+       __jump_label_transform(entry, type, text_poke_smp);
        mutex_unlock(&text_mutex);
        put_online_cpus();
 }
 
-void __init_or_module arch_jump_label_text_poke_early(jump_label_t addr)
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
 {
-       text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
-                       JUMP_LABEL_NOP_SIZE);
+       __jump_label_transform(entry, type, text_poke_early);
 }
 
 #endif

diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 66f23dc5e76a636bf2ed54542c6565fc82dbc6a4..388b0d425b507add656cca2aa7b48a346bf159ec 100644 (file)
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -16,7 +16,7 @@ struct jump_label_key {
 
 # include <asm/jump_label.h>
 # define HAVE_JUMP_LABEL
-#endif
+#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
 
 enum jump_label_type {
        JUMP_LABEL_DISABLE = 0,
@@ -28,9 +28,9 @@ struct module;
 #ifdef HAVE_JUMP_LABEL
 
 #ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
 #else
-#define JUMP_LABEL_INIT {{ 0 }, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
 #endif
 
 static __always_inline bool static_branch(struct jump_label_key *key)
@@ -41,18 +41,20 @@ static __always_inline bool static_branch(struct jump_label_key *key)
 extern struct jump_entry __start___jump_table[];
 extern struct jump_entry __stop___jump_table[];
 
+extern void jump_label_init(void);
 extern void jump_label_lock(void);
 extern void jump_label_unlock(void);
 extern void arch_jump_label_transform(struct jump_entry *entry,
-                                enum jump_label_type type);
-extern void arch_jump_label_text_poke_early(jump_label_t addr);
+                                     enum jump_label_type type);
+extern void arch_jump_label_transform_static(struct jump_entry *entry,
+                                            enum jump_label_type type);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void jump_label_inc(struct jump_label_key *key);
 extern void jump_label_dec(struct jump_label_key *key);
 extern bool jump_label_enabled(struct jump_label_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
-#else
+#else  /* !HAVE_JUMP_LABEL */
 
 #include <linux/atomic.h>
 
@@ -62,6 +64,10 @@ struct jump_label_key {
        atomic_t enabled;
 };
 
+static __always_inline void jump_label_init(void)
+{
+}
+
 static __always_inline bool static_branch(struct jump_label_key *key)
 {
        if (unlikely(atomic_read(&key->enabled)))
@@ -96,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod)
 {
        return 0;
 }
+#endif /* HAVE_JUMP_LABEL */
 
-#endif
-
-#endif
+#endif /* _LINUX_JUMP_LABEL_H */

diff --git a/init/main.c b/init/main.c
index 63f5f6f8dc3bd32c82e4b77f1082d879e98556a4..217ed23e94875d59ffd7b6a95562fcba36112e08 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -512,6 +512,9 @@ asmlinkage void __init start_kernel(void)
        parse_args("Booting kernel", static_command_line, __start___param,
                   __stop___param - __start___param,
                   &unknown_bootoption);
+
+       jump_label_init();
+
        /*
         * These use large bootmem allocations and must precede
         * kmem_cache_init()

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index a8ce45097f3d21354c7ed8c18e5fcbc4e11792d1..bbdfe2a462a088b210d5792c674b215274b1b39b 100644 (file)
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -104,6 +104,18 @@ static int __jump_label_text_reserved(struct jump_entry *iter_start,
        return 0;
 }
 
+/* 
+ * Update code which is definitely not currently executing.
+ * Architectures which need heavyweight synchronization to modify
+ * running code can override this to make the non-live update case
+ * cheaper.
+ */
+void __weak arch_jump_label_transform_static(struct jump_entry *entry,
+                                           enum jump_label_type type)
+{
+       arch_jump_label_transform(entry, type); 
+}
+
 static void __jump_label_update(struct jump_label_key *key,
                                struct jump_entry *entry,
                                struct jump_entry *stop, int enable)
@@ -121,14 +133,7 @@ static void __jump_label_update(struct jump_label_key *key,
        }
 }
 
-/*
- * Not all archs need this.
- */
-void __weak arch_jump_label_text_poke_early(jump_label_t addr)
-{
-}
-
-static __init int jump_label_init(void)
+void __init jump_label_init(void)
 {
        struct jump_entry *iter_start = __start___jump_table;
        struct jump_entry *iter_stop = __stop___jump_table;
@@ -139,22 +144,22 @@ static __init int jump_label_init(void)
        jump_label_sort_entries(iter_start, iter_stop);
 
        for (iter = iter_start; iter < iter_stop; iter++) {
-               arch_jump_label_text_poke_early(iter->code);
-               if (iter->key == (jump_label_t)(unsigned long)key)
+               struct jump_label_key *iterk;
+
+               iterk = (struct jump_label_key *)(unsigned long)iter->key;
+               arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
+                                                JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+               if (iterk == key)
                        continue;
 
-               key = (struct jump_label_key *)(unsigned long)iter->key;
-               atomic_set(&key->enabled, 0);
+               key = iterk;
                key->entries = iter;
 #ifdef CONFIG_MODULES
                key->next = NULL;
 #endif
        }
        jump_label_unlock();
-
-       return 0;
 }
-early_initcall(jump_label_init);
 
 #ifdef CONFIG_MODULES
 
@@ -212,7 +217,7 @@ void jump_label_apply_nops(struct module *mod)
                return;
 
        for (iter = iter_start; iter < iter_stop; iter++)
-               arch_jump_label_text_poke_early(iter->code);
+               arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
 }
 
 static int jump_label_add_module(struct module *mod)