X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=fs%2Fdcache.c;h=efa4602e064f75d47f064ab74084f8b81f5cc56b;hb=8134585f1c33565ea5e752a7ba9323ed3e0921f3;hp=9a59653d3449753923e3ddfd97e9d9eca2f36f04;hpb=0ae60b6553a4933483ac32247bcc0cfb313fd48f;p=firefly-linux-kernel-4.4.55.git diff --git a/fs/dcache.c b/fs/dcache.c index 9a59653d3449..efa4602e064f 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -43,7 +43,7 @@ /* * Usage: * dcache->d_inode->i_lock protects: - * - i_dentry, d_alias, d_inode of aliases + * - i_dentry, d_u.d_alias, d_inode of aliases * dcache_hash_bucket lock protects: * - the dcache hash table * s_anon bl list spinlock protects: @@ -58,7 +58,7 @@ * - d_unhashed() * - d_parent and d_subdirs * - childrens' d_child and d_parent - * - d_alias, d_inode + * - d_u.d_alias, d_inode * * Ordering: * dentry->d_inode->i_lock @@ -96,8 +96,6 @@ static struct kmem_cache *dentry_cache __read_mostly; * This hash-function tries to avoid losing too many bits of hash * information, yet avoid using a prime hash-size or similar. */ -#define D_HASHBITS d_hash_shift -#define D_HASHMASK d_hash_mask static unsigned int d_hash_mask __read_mostly; static unsigned int d_hash_shift __read_mostly; @@ -108,8 +106,7 @@ static inline struct hlist_bl_head *d_hash(const struct dentry *parent, unsigned int hash) { hash += (unsigned long) parent / L1_CACHE_BYTES; - hash = hash + (hash >> D_HASHBITS); - return dentry_hashtable + (hash & D_HASHMASK); + return dentry_hashtable + hash_32(hash, d_hash_shift); } /* Statistics gathering. */ @@ -218,7 +215,6 @@ static void __d_free(struct rcu_head *head) { struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); - WARN_ON(!hlist_unhashed(&dentry->d_alias)); if (dname_external(dentry)) kfree(dentry->d_name.name); kmem_cache_free(dentry_cache, dentry); @@ -229,6 +225,7 @@ static void __d_free(struct rcu_head *head) */ static void d_free(struct dentry *dentry) { + WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); BUG_ON(dentry->d_count); this_cpu_dec(nr_dentry); if (dentry->d_op && dentry->d_op->d_release) @@ -267,7 +264,7 @@ static void dentry_iput(struct dentry * dentry) struct inode *inode = dentry->d_inode; if (inode) { dentry->d_inode = NULL; - hlist_del_init(&dentry->d_alias); + hlist_del_init(&dentry->d_u.d_alias); spin_unlock(&dentry->d_lock); spin_unlock(&inode->i_lock); if (!inode->i_nlink) @@ -291,7 +288,7 @@ static void dentry_unlink_inode(struct dentry * dentry) { struct inode *inode = dentry->d_inode; dentry->d_inode = NULL; - hlist_del_init(&dentry->d_alias); + hlist_del_init(&dentry->d_u.d_alias); dentry_rcuwalk_barrier(dentry); spin_unlock(&dentry->d_lock); spin_unlock(&inode->i_lock); @@ -367,9 +364,9 @@ static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent) __releases(parent->d_lock) __releases(dentry->d_inode->i_lock) { - list_del(&dentry->d_u.d_child); + __list_del_entry(&dentry->d_child); /* - * Inform try_to_ascend() that we are no longer attached to the + * Inform ascending readers that we are no longer attached to the * dentry tree */ dentry->d_flags |= DCACHE_DENTRY_KILLED; @@ -663,7 +660,7 @@ static struct dentry *__d_find_alias(struct inode *inode, int want_discon) again: discon_alias = NULL; - hlist_for_each_entry(alias, &inode->i_dentry, d_alias) { + hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { spin_lock(&alias->d_lock); if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { if (IS_ROOT(alias) && @@ -716,7 +713,7 @@ void d_prune_aliases(struct inode *inode) struct dentry *dentry; restart: spin_lock(&inode->i_lock); - hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) { + hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { spin_lock(&dentry->d_lock); if (!dentry->d_count) { __dget_dlock(dentry); @@ -896,7 +893,7 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry) /* descend to the first leaf in the current subtree */ while (!list_empty(&dentry->d_subdirs)) dentry = list_entry(dentry->d_subdirs.next, - struct dentry, d_u.d_child); + struct dentry, d_child); /* consume the dentries from this leaf up through its parents * until we find one with children or run out altogether */ @@ -930,17 +927,17 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry) if (IS_ROOT(dentry)) { parent = NULL; - list_del(&dentry->d_u.d_child); + list_del(&dentry->d_child); } else { parent = dentry->d_parent; parent->d_count--; - list_del(&dentry->d_u.d_child); + list_del(&dentry->d_child); } inode = dentry->d_inode; if (inode) { dentry->d_inode = NULL; - hlist_del_init(&dentry->d_alias); + hlist_del_init(&dentry->d_u.d_alias); if (dentry->d_op && dentry->d_op->d_iput) dentry->d_op->d_iput(dentry, inode); else @@ -958,7 +955,7 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry) } while (list_empty(&dentry->d_subdirs)); dentry = list_entry(dentry->d_subdirs.next, - struct dentry, d_u.d_child); + struct dentry, d_child); } } @@ -990,35 +987,6 @@ void shrink_dcache_for_umount(struct super_block *sb) } } -/* - * This tries to ascend one level of parenthood, but - * we can race with renaming, so we need to re-check - * the parenthood after dropping the lock and check - * that the sequence number still matches. - */ -static struct dentry *try_to_ascend(struct dentry *old, int locked, unsigned seq) -{ - struct dentry *new = old->d_parent; - - rcu_read_lock(); - spin_unlock(&old->d_lock); - spin_lock(&new->d_lock); - - /* - * might go back up the wrong parent if we have had a rename - * or deletion - */ - if (new != old->d_parent || - (old->d_flags & DCACHE_DENTRY_KILLED) || - (!locked && read_seqretry(&rename_lock, seq))) { - spin_unlock(&new->d_lock); - new = NULL; - } - rcu_read_unlock(); - return new; -} - - /* * Search for at least 1 mount point in the dentry's subdirs. * We descend to the next level whenever the d_subdirs @@ -1051,7 +1019,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_child); next = tmp->next; spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); @@ -1073,30 +1041,48 @@ resume: /* * All done at this level ... ascend and resume the search. */ + rcu_read_lock(); +ascend: if (this_parent != parent) { struct dentry *child = this_parent; - this_parent = try_to_ascend(this_parent, locked, seq); - if (!this_parent) + this_parent = child->d_parent; + + spin_unlock(&child->d_lock); + spin_lock(&this_parent->d_lock); + + /* might go back up the wrong parent if we have had a rename. */ + if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; - next = child->d_u.d_child.next; + next = child->d_child.next; + while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED)) { + if (next == &this_parent->d_subdirs) + goto ascend; + child = list_entry(next, struct dentry, d_child); + next = next->next; + } + rcu_read_unlock(); goto resume; } - spin_unlock(&this_parent->d_lock); if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (locked) write_sequnlock(&rename_lock); return 0; /* No mount points found in tree */ positive: if (!locked && read_seqretry(&rename_lock, seq)) - goto rename_retry; + goto rename_retry_unlocked; if (locked) write_sequnlock(&rename_lock); return 1; rename_retry: + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (locked) goto again; +rename_retry_unlocked: locked = 1; write_seqlock(&rename_lock); goto again; @@ -1134,7 +1120,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_child); next = tmp->next; spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); @@ -1161,6 +1147,7 @@ resume: */ if (found && need_resched()) { spin_unlock(&dentry->d_lock); + rcu_read_lock(); goto out; } @@ -1180,23 +1167,40 @@ resume: /* * All done at this level ... ascend and resume the search. */ + rcu_read_lock(); +ascend: if (this_parent != parent) { struct dentry *child = this_parent; - this_parent = try_to_ascend(this_parent, locked, seq); - if (!this_parent) + this_parent = child->d_parent; + + spin_unlock(&child->d_lock); + spin_lock(&this_parent->d_lock); + + /* might go back up the wrong parent if we have had a rename. */ + if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; - next = child->d_u.d_child.next; + next = child->d_child.next; + while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED)) { + if (next == &this_parent->d_subdirs) + goto ascend; + child = list_entry(next, struct dentry, d_child); + next = next->next; + } + rcu_read_unlock(); goto resume; } out: - spin_unlock(&this_parent->d_lock); if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (locked) write_sequnlock(&rename_lock); return found; rename_retry: + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (found) return found; if (locked) @@ -1281,8 +1285,8 @@ struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) INIT_HLIST_BL_NODE(&dentry->d_hash); INIT_LIST_HEAD(&dentry->d_lru); INIT_LIST_HEAD(&dentry->d_subdirs); - INIT_HLIST_NODE(&dentry->d_alias); - INIT_LIST_HEAD(&dentry->d_u.d_child); + INIT_HLIST_NODE(&dentry->d_u.d_alias); + INIT_LIST_HEAD(&dentry->d_child); d_set_d_op(dentry, dentry->d_sb->s_d_op); this_cpu_inc(nr_dentry); @@ -1312,7 +1316,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) */ __dget_dlock(parent); dentry->d_parent = parent; - list_add(&dentry->d_u.d_child, &parent->d_subdirs); + list_add(&dentry->d_child, &parent->d_subdirs); spin_unlock(&parent->d_lock); return dentry; @@ -1372,7 +1376,7 @@ static void __d_instantiate(struct dentry *dentry, struct inode *inode) if (inode) { if (unlikely(IS_AUTOMOUNT(inode))) dentry->d_flags |= DCACHE_NEED_AUTOMOUNT; - hlist_add_head(&dentry->d_alias, &inode->i_dentry); + hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); } dentry->d_inode = inode; dentry_rcuwalk_barrier(dentry); @@ -1397,7 +1401,7 @@ static void __d_instantiate(struct dentry *dentry, struct inode *inode) void d_instantiate(struct dentry *entry, struct inode * inode) { - BUG_ON(!hlist_unhashed(&entry->d_alias)); + BUG_ON(!hlist_unhashed(&entry->d_u.d_alias)); if (inode) spin_lock(&inode->i_lock); __d_instantiate(entry, inode); @@ -1436,7 +1440,7 @@ static struct dentry *__d_instantiate_unique(struct dentry *entry, return NULL; } - hlist_for_each_entry(alias, &inode->i_dentry, d_alias) { + hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { /* * Don't need alias->d_lock here, because aliases with * d_parent == entry->d_parent are not subject to name or @@ -1462,7 +1466,7 @@ struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode) { struct dentry *result; - BUG_ON(!hlist_unhashed(&entry->d_alias)); + BUG_ON(!hlist_unhashed(&entry->d_u.d_alias)); if (inode) spin_lock(&inode->i_lock); @@ -1505,7 +1509,7 @@ static struct dentry * __d_find_any_alias(struct inode *inode) if (hlist_empty(&inode->i_dentry)) return NULL; - alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); + alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias); __dget(alias); return alias; } @@ -1579,7 +1583,7 @@ struct dentry *d_obtain_alias(struct inode *inode) spin_lock(&tmp->d_lock); tmp->d_inode = inode; tmp->d_flags |= DCACHE_DISCONNECTED; - hlist_add_head(&tmp->d_alias, &inode->i_dentry); + hlist_add_head(&tmp->d_u.d_alias, &inode->i_dentry); hlist_bl_lock(&tmp->d_sb->s_anon); hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon); hlist_bl_unlock(&tmp->d_sb->s_anon); @@ -2022,7 +2026,7 @@ int d_validate(struct dentry *dentry, struct dentry *dparent) struct dentry *child; spin_lock(&dparent->d_lock); - list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) { + list_for_each_entry(child, &dparent->d_subdirs, d_child) { if (dentry == child) { spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); __dget_dlock(dentry); @@ -2269,8 +2273,8 @@ static void __d_move(struct dentry * dentry, struct dentry * target) /* Unhash the target: dput() will then get rid of it */ __d_drop(target); - list_del(&dentry->d_u.d_child); - list_del(&target->d_u.d_child); + list_del(&dentry->d_child); + list_del(&target->d_child); /* Switch the names.. */ switch_names(dentry, target); @@ -2280,15 +2284,15 @@ static void __d_move(struct dentry * dentry, struct dentry * target) if (IS_ROOT(dentry)) { dentry->d_parent = target->d_parent; target->d_parent = target; - INIT_LIST_HEAD(&target->d_u.d_child); + INIT_LIST_HEAD(&target->d_child); } else { swap(dentry->d_parent, target->d_parent); /* And add them back to the (new) parent lists */ - list_add(&target->d_u.d_child, &target->d_parent->d_subdirs); + list_add(&target->d_child, &target->d_parent->d_subdirs); } - list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs); + list_add(&dentry->d_child, &dentry->d_parent->d_subdirs); write_seqcount_end(&target->d_seq); write_seqcount_end(&dentry->d_seq); @@ -2395,9 +2399,9 @@ static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon) swap(dentry->d_name.hash, anon->d_name.hash); dentry->d_parent = dentry; - list_del_init(&dentry->d_u.d_child); + list_del_init(&dentry->d_child); anon->d_parent = dparent; - list_move(&anon->d_u.d_child, &dparent->d_subdirs); + list_move(&anon->d_child, &dparent->d_subdirs); write_seqcount_end(&dentry->d_seq); write_seqcount_end(&anon->d_seq); @@ -2936,7 +2940,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_child); next = tmp->next; spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); @@ -2957,26 +2961,43 @@ resume: } spin_unlock(&dentry->d_lock); } + rcu_read_lock(); +ascend: if (this_parent != root) { struct dentry *child = this_parent; if (!(this_parent->d_flags & DCACHE_GENOCIDE)) { this_parent->d_flags |= DCACHE_GENOCIDE; this_parent->d_count--; } - this_parent = try_to_ascend(this_parent, locked, seq); - if (!this_parent) + this_parent = child->d_parent; + + spin_unlock(&child->d_lock); + spin_lock(&this_parent->d_lock); + + /* might go back up the wrong parent if we have had a rename. */ + if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; - next = child->d_u.d_child.next; + next = child->d_child.next; + while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED)) { + if (next == &this_parent->d_subdirs) + goto ascend; + child = list_entry(next, struct dentry, d_child); + next = next->next; + } + rcu_read_unlock(); goto resume; } - spin_unlock(&this_parent->d_lock); if (!locked && read_seqretry(&rename_lock, seq)) goto rename_retry; + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (locked) write_sequnlock(&rename_lock); return; rename_retry: + spin_unlock(&this_parent->d_lock); + rcu_read_unlock(); if (locked) goto again; locked = 1;