static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
{
- static atomic64_t id_seq = ATOMIC64_INIT(0);
struct blkcg *blkcg;
if (!parent_css) {
blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;
- blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */
done:
spin_lock_init(&blkcg->lock);
INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
struct blkcg_gq *blkg_hint;
struct hlist_head blkg_list;
- /* for policies to test whether associated blkcg has changed */
- uint64_t id;
-
/* TODO: per-policy storage in blkcg */
unsigned int cfq_weight; /* belongs to cfq */
unsigned int cfq_leaf_weight;
* @bdev: device
*
* Locates the passed device's request queue and returns the address of its
- * backing_dev_info
- *
- * Will return NULL if the request queue cannot be located.
+ * backing_dev_info. This function can only be called if @bdev is opened
+ * and the return value is never NULL.
*/
struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
{
- struct backing_dev_info *ret = NULL;
struct request_queue *q = bdev_get_queue(bdev);
- if (q)
- ret = &q->backing_dev_info;
- return ret;
+ return &q->backing_dev_info;
}
EXPORT_SYMBOL(blk_get_backing_dev_info);
* Get a free request from @q. This function may fail under memory
* pressure or if @q is dead.
*
- * Must be callled with @q->queue_lock held and,
- * Returns %NULL on failure, with @q->queue_lock held.
- * Returns !%NULL on success, with @q->queue_lock *not held*.
+ * Must be called with @q->queue_lock held and,
+ * Returns ERR_PTR on failure, with @q->queue_lock held.
+ * Returns request pointer on success, with @q->queue_lock *not held*.
*/
static struct request *__get_request(struct request_list *rl, int rw_flags,
struct bio *bio, gfp_t gfp_mask)
int may_queue;
if (unlikely(blk_queue_dying(q)))
- return NULL;
+ return ERR_PTR(-ENODEV);
may_queue = elv_may_queue(q, rw_flags);
if (may_queue == ELV_MQUEUE_NO)
* process is not a "batcher", and not
* exempted by the IO scheduler
*/
- return NULL;
+ return ERR_PTR(-ENOMEM);
}
}
}
* allocated with any setting of ->nr_requests
*/
if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
- return NULL;
+ return ERR_PTR(-ENOMEM);
q->nr_rqs[is_sync]++;
rl->count[is_sync]++;
rq_starved:
if (unlikely(rl->count[is_sync] == 0))
rl->starved[is_sync] = 1;
- return NULL;
+ return ERR_PTR(-ENOMEM);
}
/**
* Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this
* function keeps retrying under memory pressure and fails iff @q is dead.
*
- * Must be callled with @q->queue_lock held and,
- * Returns %NULL on failure, with @q->queue_lock held.
- * Returns !%NULL on success, with @q->queue_lock *not held*.
+ * Must be called with @q->queue_lock held and,
+ * Returns ERR_PTR on failure, with @q->queue_lock held.
+ * Returns request pointer on success, with @q->queue_lock *not held*.
*/
static struct request *get_request(struct request_queue *q, int rw_flags,
struct bio *bio, gfp_t gfp_mask)
rl = blk_get_rl(q, bio); /* transferred to @rq on success */
retry:
rq = __get_request(rl, rw_flags, bio, gfp_mask);
- if (rq)
+ if (!IS_ERR(rq))
return rq;
if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
blk_put_rl(rl);
- return NULL;
+ return rq;
}
/* wait on @rl and retry */
spin_lock_irq(q->queue_lock);
rq = get_request(q, rw, NULL, gfp_mask);
- if (!rq)
+ if (IS_ERR(rq))
spin_unlock_irq(q->queue_lock);
/* q->queue_lock is unlocked at this point */
{
struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
- if (unlikely(!rq))
- return ERR_PTR(-ENOMEM);
+ if (IS_ERR(rq))
+ return rq;
blk_rq_set_block_pc(rq);
* Returns with the queue unlocked.
*/
req = get_request(q, rw_flags, bio, GFP_NOIO);
- if (unlikely(!req)) {
- bio_endio(bio, -ENODEV); /* @q is dead */
+ if (IS_ERR(req)) {
+ bio_endio(bio, PTR_ERR(req)); /* @q is dead */
goto out_unlock;
}
struct blk_mq_hw_ctx *hctx;
struct request *rq;
struct blk_mq_alloc_data alloc_data;
+ int ret;
- if (blk_mq_queue_enter(q))
- return NULL;
+ ret = blk_mq_queue_enter(q);
+ if (ret)
+ return ERR_PTR(ret);
ctx = blk_mq_get_ctx(q);
hctx = q->mq_ops->map_queue(q, ctx->cpu);
ctx = alloc_data.ctx;
}
blk_mq_put_ctx(ctx);
+ if (!rq)
+ return ERR_PTR(-EWOULDBLOCK);
return rq;
}
EXPORT_SYMBOL(blk_mq_alloc_request);
* map scatter-gather elements separately and string them to request
*/
rq = blk_get_request(q, rw, GFP_KERNEL);
- if (!rq)
- return ERR_PTR(-ENOMEM);
+ if (IS_ERR(rq))
+ return rq;
blk_rq_set_block_pc(rq);
ret = blk_fill_sgv4_hdr_rq(q, rq, hdr, bd, has_write_perm);
}
next_rq = blk_get_request(q, READ, GFP_KERNEL);
- if (!next_rq) {
- ret = -ENOMEM;
+ if (IS_ERR(next_rq)) {
+ ret = PTR_ERR(next_rq);
+ next_rq = NULL;
goto out;
}
rq->next_rq = next_rq;
struct cfq_ttime ttime;
int ioprio; /* the current ioprio */
#ifdef CONFIG_CFQ_GROUP_IOSCHED
- uint64_t blkcg_id; /* the current blkcg ID */
+ uint64_t blkcg_serial_nr; /* the current blkcg serial */
#endif
};
{
struct cfq_data *cfqd = cic_to_cfqd(cic);
struct cfq_queue *sync_cfqq;
- uint64_t id;
+ uint64_t serial_nr;
rcu_read_lock();
- id = bio_blkcg(bio)->id;
+ serial_nr = bio_blkcg(bio)->css.serial_nr;
rcu_read_unlock();
/*
* Check whether blkcg has changed. The condition may trigger
* spuriously on a newly created cic but there's no harm.
*/
- if (unlikely(!cfqd) || likely(cic->blkcg_id == id))
+ if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr))
return;
sync_cfqq = cic_to_cfqq(cic, 1);
cfq_put_queue(sync_cfqq);
}
- cic->blkcg_id = id;
+ cic->blkcg_serial_nr = serial_nr;
}
#else
static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { }
if (!arg)
return -EINVAL;
bdi = blk_get_backing_dev_info(bdev);
- if (bdi == NULL)
- return -ENOTTY;
return compat_put_long(arg,
(bdi->ra_pages * PAGE_CACHE_SIZE) / 512);
case BLKROGET: /* compatible */
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
bdi = blk_get_backing_dev_info(bdev);
- if (bdi == NULL)
- return -ENOTTY;
bdi->ra_pages = (arg * 512) / PAGE_CACHE_SIZE;
return 0;
case BLKGETSIZE:
if (!arg)
return -EINVAL;
bdi = blk_get_backing_dev_info(bdev);
- if (bdi == NULL)
- return -ENOTTY;
return put_long(arg, (bdi->ra_pages * PAGE_CACHE_SIZE) / 512);
case BLKROGET:
return put_int(arg, bdev_read_only(bdev) != 0);
if(!capable(CAP_SYS_ADMIN))
return -EACCES;
bdi = blk_get_backing_dev_info(bdev);
- if (bdi == NULL)
- return -ENOTTY;
bdi->ra_pages = (arg * 512) / PAGE_CACHE_SIZE;
return 0;
case BLKBSZSET:
ret = -ENOMEM;
rq = blk_get_request(q, writing ? WRITE : READ, GFP_KERNEL);
- if (!rq)
- goto out;
+ if (IS_ERR(rq))
+ return PTR_ERR(rq);
blk_rq_set_block_pc(rq);
if (hdr->cmd_len > BLK_MAX_CDB) {
kfree(rq->cmd);
out_put_request:
blk_put_request(rq);
-out:
return ret;
}
}
rq = blk_get_request(q, in_len ? WRITE : READ, __GFP_WAIT);
- if (!rq) {
- err = -ENOMEM;
+ if (IS_ERR(rq)) {
+ err = PTR_ERR(rq);
goto error;
}
blk_rq_set_block_pc(rq);
int err;
rq = blk_get_request(q, WRITE, __GFP_WAIT);
+ if (IS_ERR(rq))
+ return PTR_ERR(rq);
blk_rq_set_block_pc(rq);
rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
rq->cmd[0] = cmd;
int err = 0;
rq = blk_get_request(disk->gd->queue, READ, __GFP_WAIT);
+ if (IS_ERR(rq))
+ return PTR_ERR(rq);
rq->cmd_type = REQ_TYPE_SPECIAL;
rq->special = func;
rq = blk_get_request(q, (cgc->data_direction == CGC_DATA_WRITE) ?
WRITE : READ, __GFP_WAIT);
+ if (IS_ERR(rq))
+ return PTR_ERR(rq);
blk_rq_set_block_pc(rq);
if (cgc->buflen) {
return NULL;
rq = blk_get_request(host->oob_q, WRITE /* bogus */, GFP_KERNEL);
- if (!rq) {
+ if (IS_ERR(rq)) {
spin_lock_irqsave(&host->lock, flags);
carm_put_request(host, crq);
spin_unlock_irqrestore(&host->lock, flags);
len = nr * CD_FRAMESIZE_RAW;
rq = blk_get_request(q, READ, GFP_KERNEL);
- if (!rq) {
- ret = -ENOMEM;
+ if (IS_ERR(rq)) {
+ ret = PTR_ERR(rq);
break;
}
blk_rq_set_block_pc(rq);
* timeout has expired, so power management will be reenabled.
*/
rq = blk_get_request(q, READ, GFP_NOWAIT);
- if (unlikely(!rq))
+ if (IS_ERR(rq))
goto out;
rq->cmd[0] = REQ_UNPARK_HEADS;
rq = blk_get_request(q, rw, GFP_NOIO);
- if (!rq) {
+ if (IS_ERR(rq)) {
sdev_printk(KERN_INFO, sdev,
"%s: blk_get_request failed\n", __func__);
return NULL;
rq = blk_get_request(sdev->request_queue,
(cmd != INQUIRY) ? WRITE : READ, GFP_NOIO);
- if (!rq) {
+ if (IS_ERR(rq)) {
sdev_printk(KERN_INFO, sdev, "get_req: blk_get_request failed");
return NULL;
}
retry:
req = blk_get_request(sdev->request_queue, WRITE, GFP_NOIO);
- if (!req)
+ if (IS_ERR(req))
return SCSI_DH_RES_TEMP_UNAVAIL;
blk_rq_set_block_pc(req);
struct request *req;
req = blk_get_request(h->sdev->request_queue, WRITE, GFP_ATOMIC);
- if (!req)
+ if (IS_ERR(req))
return SCSI_DH_RES_TEMP_UNAVAIL;
blk_rq_set_block_pc(req);
rq = blk_get_request(q, rw, GFP_NOIO);
- if (!rq) {
+ if (IS_ERR(rq)) {
sdev_printk(KERN_INFO, sdev,
"get_rdac_req: blk_get_request failed.\n");
return NULL;
struct request *req;
req = blk_get_request(q, has_write ? WRITE : READ, flags);
- if (unlikely(!req))
- return ERR_PTR(-ENOMEM);
+ if (IS_ERR(req))
+ return req;
blk_rq_set_block_pc(req);
return req;
int write = (data_direction == DMA_TO_DEVICE);
req = blk_get_request(SRpnt->stp->device->request_queue, write, GFP_KERNEL);
- if (!req)
+ if (IS_ERR(req))
return DRIVER_ERROR << 24;
blk_rq_set_block_pc(req);
* request becomes available
*/
req = blk_get_request(sdev->request_queue, READ, GFP_KERNEL);
+ if (IS_ERR(req))
+ return;
blk_rq_set_block_pc(req);
int ret = DRIVER_ERROR << 24;
req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
- if (!req)
+ if (IS_ERR(req))
return ret;
blk_rq_set_block_pc(req);
}
rq = blk_get_request(q, rw, GFP_ATOMIC);
- if (!rq) {
+ if (IS_ERR(rq)) {
kfree(long_cmdp);
- return -ENOMEM;
+ return PTR_ERR(rq);
}
blk_rq_set_block_pc(rq);
req = blk_get_request(SRpnt->stp->device->request_queue, write,
GFP_KERNEL);
- if (!req)
+ if (IS_ERR(req))
return DRIVER_ERROR << 24;
blk_rq_set_block_pc(req);
req = blk_get_request(pdv->pdv_sd->request_queue,
(data_direction == DMA_TO_DEVICE),
GFP_KERNEL);
- if (!req) {
+ if (IS_ERR(req)) {
pr_err("PSCSI: blk_get_request() failed\n");
ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
goto fail;
EXPORT_SYMBOL(I_BDEV);
/*
- * Move the inode from its current bdi to a new bdi. If the inode is dirty we
- * need to move it onto the dirty list of @dst so that the inode is always on
- * the right list.
+ * Move the inode from its current bdi to a new bdi. Make sure the inode
+ * is clean before moving so that it doesn't linger on the old bdi.
*/
static void bdev_inode_switch_bdi(struct inode *inode,
struct backing_dev_info *dst)
{
- struct backing_dev_info *old = inode->i_data.backing_dev_info;
- bool wakeup_bdi = false;
-
- if (unlikely(dst == old)) /* deadlock avoidance */
- return;
- bdi_lock_two(&old->wb, &dst->wb);
- spin_lock(&inode->i_lock);
- inode->i_data.backing_dev_info = dst;
- if (inode->i_state & I_DIRTY) {
- if (bdi_cap_writeback_dirty(dst) && !wb_has_dirty_io(&dst->wb))
- wakeup_bdi = true;
- list_move(&inode->i_wb_list, &dst->wb.b_dirty);
+ while (true) {
+ spin_lock(&inode->i_lock);
+ if (!(inode->i_state & I_DIRTY)) {
+ inode->i_data.backing_dev_info = dst;
+ spin_unlock(&inode->i_lock);
+ return;
+ }
+ spin_unlock(&inode->i_lock);
+ WARN_ON_ONCE(write_inode_now(inode, true));
}
- spin_unlock(&inode->i_lock);
- spin_unlock(&old->wb.list_lock);
- spin_unlock(&dst->wb.list_lock);
-
- if (wakeup_bdi)
- bdi_wakeup_thread_delayed(dst);
}
/* Kill _all_ buffers and pagecache , dirty or not.. */
if (!ret) {
bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
bdi = blk_get_backing_dev_info(bdev);
- if (bdi == NULL)
- bdi = &default_backing_dev_info;
bdev_inode_switch_bdi(bdev->bd_inode, bdi);
}
if (!device->bdev)
continue;
bdi = blk_get_backing_dev_info(device->bdev);
- if (bdi && bdi_congested(bdi, bdi_bits)) {
+ if (bdi_congested(bdi, bdi_bits)) {
ret = 1;
break;
}
btp->bt_dev = bdev->bd_dev;
btp->bt_bdev = bdev;
btp->bt_bdi = blk_get_backing_dev_info(bdev);
- if (!btp->bt_bdi)
- goto error;
if (xfs_setsize_buftarg_early(btp, bdev))
goto error;
* Bits in backing_dev_info.state
*/
enum bdi_state {
- BDI_wb_alloc, /* Default embedded wb allocated */
BDI_async_congested, /* The async (write) queue is getting full */
BDI_sync_congested, /* The sync queue is getting full */
BDI_registered, /* bdi_register() was done */
BDI_writeback_running, /* Writeback is in progress */
- BDI_unused, /* Available bits start here */
};
typedef int (congested_fn)(void *, int);
struct bdi_writeback {
struct backing_dev_info *bdi; /* our parent bdi */
- unsigned int nr;
unsigned long last_old_flush; /* last old data flush */
void bdi_writeback_workfn(struct work_struct *work);
int bdi_has_dirty_io(struct backing_dev_info *bdi);
void bdi_wakeup_thread_delayed(struct backing_dev_info *bdi);
-void bdi_lock_two(struct bdi_writeback *wb1, struct bdi_writeback *wb2);
extern spinlock_t bdi_lock;
extern struct list_head bdi_list;
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
{
- return bdev->bd_disk->queue;
+ return bdev->bd_disk->queue; /* this is never NULL */
}
/*
/* bdi_wq serves all asynchronous writeback tasks */
struct workqueue_struct *bdi_wq;
-void bdi_lock_two(struct bdi_writeback *wb1, struct bdi_writeback *wb2)
+static void bdi_lock_two(struct bdi_writeback *wb1, struct bdi_writeback *wb2)
{
if (wb1 < wb2) {
spin_lock(&wb1->list_lock);
mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
flush_delayed_work(&bdi->wb.dwork);
WARN_ON(!list_empty(&bdi->work_list));
-
- /*
- * This shouldn't be necessary unless @bdi for some reason has
- * unflushed dirty IO after work_list is drained. Do it anyway
- * just in case.
- */
- cancel_delayed_work_sync(&bdi->wb.dwork);
+ WARN_ON(delayed_work_pending(&bdi->wb.dwork));
}
/*
void bdi_unregister(struct backing_dev_info *bdi)
{
- struct device *dev = bdi->dev;
-
- if (dev) {
+ if (bdi->dev) {
bdi_set_min_ratio(bdi, 0);
trace_writeback_bdi_unregister(bdi);
bdi_prune_sb(bdi);
bdi_wb_shutdown(bdi);
bdi_debug_unregister(bdi);
-
- spin_lock_bh(&bdi->wb_lock);
+ device_unregister(bdi->dev);
bdi->dev = NULL;
- spin_unlock_bh(&bdi->wb_lock);
-
- device_unregister(dev);
}
}
EXPORT_SYMBOL(bdi_unregister);
int i;
/*
- * Splice our entries to the default_backing_dev_info, if this
- * bdi disappears
+ * Splice our entries to the default_backing_dev_info. This
+ * condition shouldn't happen. @wb must be empty at this point and
+ * dirty inodes on it might cause other issues. This workaround is
+ * added by ce5f8e779519 ("writeback: splice dirty inode entries to
+ * default bdi on bdi_destroy()") without root-causing the issue.
+ *
+ * http://lkml.kernel.org/g/1253038617-30204-11-git-send-email-jens.axboe@oracle.com
+ * http://thread.gmane.org/gmane.linux.file-systems/35341/focus=35350
+ *
+ * We should probably add WARN_ON() to find out whether it still
+ * happens and track it down if so.
*/
if (bdi_has_dirty_io(bdi)) {
struct bdi_writeback *dst = &default_backing_dev_info.wb;
bdi_unregister(bdi);
- /*
- * If bdi_unregister() had already been called earlier, the dwork
- * could still be pending because bdi_prune_sb() can race with the
- * bdi_wakeup_thread_delayed() calls from __mark_inode_dirty().
- */
- cancel_delayed_work_sync(&bdi->wb.dwork);
+ WARN_ON(delayed_work_pending(&bdi->wb.dwork));
for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
percpu_counter_destroy(&bdi->bdi_stat[i]);
projects / firefly-linux-kernel-4.4.55.git / commitdiff