1 #include <linux/module.h>
2 #include <linux/moduleparam.h>
3 #include <linux/sched.h>
5 #include <linux/blkdev.h>
6 #include <linux/init.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/hrtimer.h>
12 struct list_head list;
13 struct llist_node ll_list;
14 struct call_single_data csd;
18 struct nullb_queue *nq;
22 unsigned long *tag_map;
23 wait_queue_head_t wait;
24 unsigned int queue_depth;
26 struct nullb_cmd *cmds;
30 struct list_head list;
32 struct request_queue *q;
35 unsigned int queue_depth;
38 struct nullb_queue *queues;
39 unsigned int nr_queues;
42 static LIST_HEAD(nullb_list);
43 static struct mutex lock;
44 static int null_major;
45 static int nullb_indexes;
47 struct completion_queue {
48 struct llist_head list;
53 * These are per-cpu for now, they will need to be configured by the
54 * complete_queues parameter and appropriately mapped.
56 static DEFINE_PER_CPU(struct completion_queue, completion_queues);
68 static int submit_queues = 1;
69 module_param(submit_queues, int, S_IRUGO);
70 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
72 static int home_node = NUMA_NO_NODE;
73 module_param(home_node, int, S_IRUGO);
74 MODULE_PARM_DESC(home_node, "Home node for the device");
76 static int queue_mode = NULL_Q_MQ;
77 module_param(queue_mode, int, S_IRUGO);
78 MODULE_PARM_DESC(use_mq, "Use blk-mq interface (0=bio,1=rq,2=multiqueue)");
81 module_param(gb, int, S_IRUGO);
82 MODULE_PARM_DESC(gb, "Size in GB");
85 module_param(bs, int, S_IRUGO);
86 MODULE_PARM_DESC(bs, "Block size (in bytes)");
88 static int nr_devices = 2;
89 module_param(nr_devices, int, S_IRUGO);
90 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
92 static int irqmode = NULL_IRQ_SOFTIRQ;
93 module_param(irqmode, int, S_IRUGO);
94 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
96 static int completion_nsec = 10000;
97 module_param(completion_nsec, int, S_IRUGO);
98 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
100 static int hw_queue_depth = 64;
101 module_param(hw_queue_depth, int, S_IRUGO);
102 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
104 static bool use_per_node_hctx = true;
105 module_param(use_per_node_hctx, bool, S_IRUGO);
106 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: true");
108 static void put_tag(struct nullb_queue *nq, unsigned int tag)
110 clear_bit_unlock(tag, nq->tag_map);
112 if (waitqueue_active(&nq->wait))
116 static unsigned int get_tag(struct nullb_queue *nq)
121 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
122 if (tag >= nq->queue_depth)
124 } while (test_and_set_bit_lock(tag, nq->tag_map));
129 static void free_cmd(struct nullb_cmd *cmd)
131 put_tag(cmd->nq, cmd->tag);
134 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
136 struct nullb_cmd *cmd;
141 cmd = &nq->cmds[tag];
150 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
152 struct nullb_cmd *cmd;
155 cmd = __alloc_cmd(nq);
156 if (cmd || !can_wait)
160 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
161 cmd = __alloc_cmd(nq);
168 finish_wait(&nq->wait, &wait);
172 static void end_cmd(struct nullb_cmd *cmd)
175 if (queue_mode == NULL_Q_MQ)
176 blk_mq_end_io(cmd->rq, 0);
178 INIT_LIST_HEAD(&cmd->rq->queuelist);
179 blk_end_request_all(cmd->rq, 0);
182 bio_endio(cmd->bio, 0);
184 if (queue_mode != NULL_Q_MQ)
188 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
190 struct completion_queue *cq;
191 struct llist_node *entry;
192 struct nullb_cmd *cmd;
194 cq = &per_cpu(completion_queues, smp_processor_id());
196 while ((entry = llist_del_all(&cq->list)) != NULL) {
198 cmd = container_of(entry, struct nullb_cmd, ll_list);
204 return HRTIMER_NORESTART;
207 static void null_cmd_end_timer(struct nullb_cmd *cmd)
209 struct completion_queue *cq = &per_cpu(completion_queues, get_cpu());
211 cmd->ll_list.next = NULL;
212 if (llist_add(&cmd->ll_list, &cq->list)) {
213 ktime_t kt = ktime_set(0, completion_nsec);
215 hrtimer_start(&cq->timer, kt, HRTIMER_MODE_REL);
221 static void null_softirq_done_fn(struct request *rq)
223 blk_end_request_all(rq, 0);
228 static void null_ipi_cmd_end_io(void *data)
230 struct completion_queue *cq;
231 struct llist_node *entry, *next;
232 struct nullb_cmd *cmd;
234 cq = &per_cpu(completion_queues, smp_processor_id());
236 entry = llist_del_all(&cq->list);
240 cmd = llist_entry(entry, struct nullb_cmd, ll_list);
246 static void null_cmd_end_ipi(struct nullb_cmd *cmd)
248 struct call_single_data *data = &cmd->csd;
250 struct completion_queue *cq = &per_cpu(completion_queues, cpu);
252 cmd->ll_list.next = NULL;
254 if (llist_add(&cmd->ll_list, &cq->list)) {
255 data->func = null_ipi_cmd_end_io;
257 __smp_call_function_single(cpu, data, 0);
263 #endif /* CONFIG_SMP */
265 static inline void null_handle_cmd(struct nullb_cmd *cmd)
267 /* Complete IO by inline, softirq or timer */
272 case NULL_IRQ_SOFTIRQ:
274 null_cmd_end_ipi(cmd);
280 null_cmd_end_timer(cmd);
285 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
289 if (nullb->nr_queues != 1)
290 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
292 return &nullb->queues[index];
295 static void null_queue_bio(struct request_queue *q, struct bio *bio)
297 struct nullb *nullb = q->queuedata;
298 struct nullb_queue *nq = nullb_to_queue(nullb);
299 struct nullb_cmd *cmd;
301 cmd = alloc_cmd(nq, 1);
304 null_handle_cmd(cmd);
307 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
309 struct nullb *nullb = q->queuedata;
310 struct nullb_queue *nq = nullb_to_queue(nullb);
311 struct nullb_cmd *cmd;
313 cmd = alloc_cmd(nq, 0);
320 return BLKPREP_DEFER;
323 static void null_request_fn(struct request_queue *q)
327 while ((rq = blk_fetch_request(q)) != NULL) {
328 struct nullb_cmd *cmd = rq->special;
330 spin_unlock_irq(q->queue_lock);
331 null_handle_cmd(cmd);
332 spin_lock_irq(q->queue_lock);
336 static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq)
338 struct nullb_cmd *cmd = rq->special;
341 cmd->nq = hctx->driver_data;
343 null_handle_cmd(cmd);
344 return BLK_MQ_RQ_QUEUE_OK;
347 static struct blk_mq_hw_ctx *null_alloc_hctx(struct blk_mq_reg *reg, unsigned int hctx_index)
349 return kzalloc_node(sizeof(struct blk_mq_hw_ctx), GFP_KERNEL,
353 static void null_free_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_index)
358 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
361 struct nullb *nullb = data;
362 struct nullb_queue *nq = &nullb->queues[index];
364 init_waitqueue_head(&nq->wait);
365 nq->queue_depth = nullb->queue_depth;
367 hctx->driver_data = nq;
372 static struct blk_mq_ops null_mq_ops = {
373 .queue_rq = null_queue_rq,
374 .map_queue = blk_mq_map_queue,
375 .init_hctx = null_init_hctx,
378 static struct blk_mq_reg null_mq_reg = {
381 .cmd_size = sizeof(struct nullb_cmd),
382 .flags = BLK_MQ_F_SHOULD_MERGE,
385 static void null_del_dev(struct nullb *nullb)
387 list_del_init(&nullb->list);
389 del_gendisk(nullb->disk);
390 if (queue_mode == NULL_Q_MQ)
391 blk_mq_free_queue(nullb->q);
393 blk_cleanup_queue(nullb->q);
394 put_disk(nullb->disk);
398 static int null_open(struct block_device *bdev, fmode_t mode)
403 static void null_release(struct gendisk *disk, fmode_t mode)
407 static const struct block_device_operations null_fops = {
408 .owner = THIS_MODULE,
410 .release = null_release,
413 static int setup_commands(struct nullb_queue *nq)
415 struct nullb_cmd *cmd;
418 nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
422 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
423 nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
429 for (i = 0; i < nq->queue_depth; i++) {
431 INIT_LIST_HEAD(&cmd->list);
432 cmd->ll_list.next = NULL;
439 static void cleanup_queue(struct nullb_queue *nq)
445 static void cleanup_queues(struct nullb *nullb)
449 for (i = 0; i < nullb->nr_queues; i++)
450 cleanup_queue(&nullb->queues[i]);
452 kfree(nullb->queues);
455 static int setup_queues(struct nullb *nullb)
457 struct nullb_queue *nq;
460 nullb->queues = kzalloc(submit_queues * sizeof(*nq), GFP_KERNEL);
464 nullb->nr_queues = 0;
465 nullb->queue_depth = hw_queue_depth;
467 if (queue_mode == NULL_Q_MQ)
470 for (i = 0; i < submit_queues; i++) {
471 nq = &nullb->queues[i];
472 init_waitqueue_head(&nq->wait);
473 nq->queue_depth = hw_queue_depth;
474 if (setup_commands(nq))
479 if (i == submit_queues)
482 cleanup_queues(nullb);
486 static int null_add_dev(void)
488 struct gendisk *disk;
492 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
496 spin_lock_init(&nullb->lock);
498 if (setup_queues(nullb))
501 if (queue_mode == NULL_Q_MQ) {
502 null_mq_reg.numa_node = home_node;
503 null_mq_reg.queue_depth = hw_queue_depth;
505 if (use_per_node_hctx) {
506 null_mq_reg.ops->alloc_hctx = null_alloc_hctx;
507 null_mq_reg.ops->free_hctx = null_free_hctx;
509 null_mq_reg.nr_hw_queues = nr_online_nodes;
511 null_mq_reg.ops->alloc_hctx = blk_mq_alloc_single_hw_queue;
512 null_mq_reg.ops->free_hctx = blk_mq_free_single_hw_queue;
514 null_mq_reg.nr_hw_queues = submit_queues;
517 nullb->q = blk_mq_init_queue(&null_mq_reg, nullb);
518 } else if (queue_mode == NULL_Q_BIO) {
519 nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
520 blk_queue_make_request(nullb->q, null_queue_bio);
522 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
523 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
525 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
531 nullb->q->queuedata = nullb;
532 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
534 disk = nullb->disk = alloc_disk_node(1, home_node);
537 if (queue_mode == NULL_Q_MQ)
538 blk_mq_free_queue(nullb->q);
540 blk_cleanup_queue(nullb->q);
541 cleanup_queues(nullb);
548 list_add_tail(&nullb->list, &nullb_list);
549 nullb->index = nullb_indexes++;
552 blk_queue_logical_block_size(nullb->q, bs);
553 blk_queue_physical_block_size(nullb->q, bs);
555 size = gb * 1024 * 1024 * 1024ULL;
556 sector_div(size, bs);
557 set_capacity(disk, size);
559 disk->flags |= GENHD_FL_EXT_DEVT;
560 disk->major = null_major;
561 disk->first_minor = nullb->index;
562 disk->fops = &null_fops;
563 disk->private_data = nullb;
564 disk->queue = nullb->q;
565 sprintf(disk->disk_name, "nullb%d", nullb->index);
570 static int __init null_init(void)
574 #if !defined(CONFIG_SMP)
575 if (irqmode == NULL_IRQ_SOFTIRQ) {
576 pr_warn("null_blk: softirq completions not available.\n");
577 pr_warn("null_blk: using direct completions.\n");
578 irqmode = NULL_IRQ_NONE;
582 if (submit_queues > nr_cpu_ids)
583 submit_queues = nr_cpu_ids;
584 else if (!submit_queues)
589 /* Initialize a separate list for each CPU for issuing softirqs */
590 for_each_possible_cpu(i) {
591 struct completion_queue *cq = &per_cpu(completion_queues, i);
593 init_llist_head(&cq->list);
595 if (irqmode != NULL_IRQ_TIMER)
598 hrtimer_init(&cq->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
599 cq->timer.function = null_cmd_timer_expired;
602 null_major = register_blkdev(0, "nullb");
606 for (i = 0; i < nr_devices; i++) {
607 if (null_add_dev()) {
608 unregister_blkdev(null_major, "nullb");
613 pr_info("null: module loaded\n");
617 static void __exit null_exit(void)
621 unregister_blkdev(null_major, "nullb");
624 while (!list_empty(&nullb_list)) {
625 nullb = list_entry(nullb_list.next, struct nullb, list);
631 module_init(null_init);
632 module_exit(null_exit);
634 MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
635 MODULE_LICENSE("GPL");