raid5: reduce chance release_stripe() taking device_lock

release_stripe() is a place conf->device_lock is heavily contended. We take the
lock even stripe count isn't 1, which isn't required.

Signed-off-by: Shaohua Li <shli@fusionio.com>
Signed-off-by: NeilBrown <neilb@suse.de>

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 04348d76bb30fa8831964ea980ec2df912a45f92..848034666342a0e9b8ec744955d4659ee6b5f48f 100644 (file)
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -190,49 +190,56 @@ static int stripe_operations_active(struct stripe_head *sh)
               test_bit(STRIPE_COMPUTE_RUN, &sh->state);
 }
 
-static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
-{
-       if (atomic_dec_and_test(&sh->count)) {
-               BUG_ON(!list_empty(&sh->lru));
-               BUG_ON(atomic_read(&conf->active_stripes)==0);
-               if (test_bit(STRIPE_HANDLE, &sh->state)) {
-                       if (test_bit(STRIPE_DELAYED, &sh->state) &&
-                           !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-                               list_add_tail(&sh->lru, &conf->delayed_list);
-                       else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
-                                  sh->bm_seq - conf->seq_write > 0)
-                               list_add_tail(&sh->lru, &conf->bitmap_list);
-                       else {
-                               clear_bit(STRIPE_DELAYED, &sh->state);
-                               clear_bit(STRIPE_BIT_DELAY, &sh->state);
-                               list_add_tail(&sh->lru, &conf->handle_list);
-                       }
-                       md_wakeup_thread(conf->mddev->thread);
-               } else {
-                       BUG_ON(stripe_operations_active(sh));
-                       if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-                               if (atomic_dec_return(&conf->preread_active_stripes)
-                                   < IO_THRESHOLD)
-                                       md_wakeup_thread(conf->mddev->thread);
-                       atomic_dec(&conf->active_stripes);
-                       if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
-                               list_add_tail(&sh->lru, &conf->inactive_list);
-                               wake_up(&conf->wait_for_stripe);
-                               if (conf->retry_read_aligned)
-                                       md_wakeup_thread(conf->mddev->thread);
-                       }
+static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh)
+{
+       BUG_ON(!list_empty(&sh->lru));
+       BUG_ON(atomic_read(&conf->active_stripes)==0);
+       if (test_bit(STRIPE_HANDLE, &sh->state)) {
+               if (test_bit(STRIPE_DELAYED, &sh->state) &&
+                   !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                       list_add_tail(&sh->lru, &conf->delayed_list);
+               else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
+                          sh->bm_seq - conf->seq_write > 0)
+                       list_add_tail(&sh->lru, &conf->bitmap_list);
+               else {
+                       clear_bit(STRIPE_DELAYED, &sh->state);
+                       clear_bit(STRIPE_BIT_DELAY, &sh->state);
+                       list_add_tail(&sh->lru, &conf->handle_list);
+               }
+               md_wakeup_thread(conf->mddev->thread);
+       } else {
+               BUG_ON(stripe_operations_active(sh));
+               if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                       if (atomic_dec_return(&conf->preread_active_stripes)
+                           < IO_THRESHOLD)
+                               md_wakeup_thread(conf->mddev->thread);
+               atomic_dec(&conf->active_stripes);
+               if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
+                       list_add_tail(&sh->lru, &conf->inactive_list);
+                       wake_up(&conf->wait_for_stripe);
+                       if (conf->retry_read_aligned)
+                               md_wakeup_thread(conf->mddev->thread);
                }
        }
 }
 
+static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
+{
+       if (atomic_dec_and_test(&sh->count))
+               do_release_stripe(conf, sh);
+}
+
 static void release_stripe(struct stripe_head *sh)
 {
        struct r5conf *conf = sh->raid_conf;
        unsigned long flags;
 
-       spin_lock_irqsave(&conf->device_lock, flags);
-       __release_stripe(conf, sh);
-       spin_unlock_irqrestore(&conf->device_lock, flags);
+       local_irq_save(flags);
+       if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
+               do_release_stripe(conf, sh);
+               spin_unlock(&conf->device_lock);
+       }
+       local_irq_restore(flags);
 }
 
 static inline void remove_hash(struct stripe_head *sh)