drivers/video/tegra/host/nvhost_intr.c

   1 /*
   2  * drivers/video/tegra/host/nvhost_intr.c
   3  *
   4  * Tegra Graphics Host Interrupt Management
   5  *
   6  * Copyright (c) 2010, NVIDIA Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #include "nvhost_intr.h"
  24 #include "dev.h"
  25 #include <linux/interrupt.h>
  26 #include <linux/slab.h>
  27 #include <linux/irq.h>
  28
  29 #define intr_to_dev(x) container_of(x, struct nvhost_master, intr)
  30
  31
  32 /*** HW sync point threshold interrupt management ***/
  33
  34 static void set_syncpt_threshold(void __iomem *sync_regs, u32 id, u32 thresh)
  35 {
  36         thresh &= 0xffff;
  37         writel(thresh, sync_regs + (HOST1X_SYNC_SYNCPT_INT_THRESH_0 + id * 4));
  38 }
  39
  40 static void enable_syncpt_interrupt(void __iomem *sync_regs, u32 id)
  41 {
  42         writel(BIT(id), sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0);
  43 }
  44
  45
  46 /*** Wait list management ***/
  47
  48 struct nvhost_waitlist {
  49         struct list_head list;
  50         struct kref refcount;
  51         u32 thresh;
  52         enum nvhost_intr_action action;
  53         atomic_t state;
  54         void *data;
  55         int count;
  56 };
  57
  58 enum waitlist_state
  59 {
  60         WLS_PENDING,
  61         WLS_REMOVED,
  62         WLS_CANCELLED,
  63         WLS_HANDLED
  64 };
  65
  66 static void waiter_release(struct kref *kref)
  67 {
  68         kfree(container_of(kref, struct nvhost_waitlist, refcount));
  69 }
  70
  71 /*
  72  * add a waiter to a waiter queue, sorted by threshold
  73  * returns true if it was added at the head of the queue
  74  */
  75 static bool add_waiter_to_queue(struct nvhost_waitlist *waiter,
  76                                 struct list_head *queue)
  77 {
  78         struct nvhost_waitlist *pos;
  79         u32 thresh = waiter->thresh;
  80
  81         list_for_each_entry_reverse(pos, queue, list)
  82                 if ((s32)(pos->thresh - thresh) <= 0) {
  83                         list_add(&waiter->list, &pos->list);
  84                         return false;
  85                 }
  86
  87         list_add(&waiter->list, queue);
  88         return true;
  89 }
  90
  91 /*
  92  * run through a waiter queue for a single sync point ID
  93  * and gather all completed waiters into lists by actions
  94  */
  95 static void remove_completed_waiters(struct list_head *head, u32 sync,
  96                         struct list_head completed[NVHOST_INTR_ACTION_COUNT])
  97 {
  98         struct list_head *dest;
  99         struct nvhost_waitlist *waiter, *next, *prev;
 100
 101         list_for_each_entry_safe(waiter, next, head, list) {
 102                 if ((s32)(waiter->thresh - sync) > 0)
 103                         break;
 104
 105                 dest = completed + waiter->action;
 106
 107                 /* consolidate submit cleanups */
 108                 if (waiter->action == NVHOST_INTR_ACTION_SUBMIT_COMPLETE
 109                         && !list_empty(dest)) {
 110                         prev = list_entry(dest->prev,
 111                                         struct nvhost_waitlist, list);
 112                         if (prev->data == waiter->data) {
 113                                 prev->count++;
 114                                 dest = NULL;
 115                         }
 116                 }
 117
 118                 /* PENDING->REMOVED or CANCELLED->HANDLED */
 119                 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
 120                         list_del(&waiter->list);
 121                         kref_put(&waiter->refcount, waiter_release);
 122                 } else {
 123                         list_move_tail(&waiter->list, dest);
 124                 }
 125         }
 126 }
 127
 128 static void action_submit_complete(struct nvhost_waitlist *waiter)
 129 {
 130         struct nvhost_channel *channel = waiter->data;
 131         int nr_completed = waiter->count;
 132
 133         nvhost_cdma_update(&channel->cdma);
 134         nvhost_module_idle_mult(&channel->mod, nr_completed);
 135 }
 136
 137 static void action_ctxsave(struct nvhost_waitlist *waiter)
 138 {
 139         struct nvhost_hwctx *hwctx = waiter->data;
 140         struct nvhost_channel *channel = hwctx->channel;
 141
 142         channel->ctxhandler.save_service(hwctx);
 143         channel->ctxhandler.put(hwctx);
 144 }
 145
 146 static void action_wakeup(struct nvhost_waitlist *waiter)
 147 {
 148         wait_queue_head_t *wq = waiter->data;
 149
 150         wake_up(wq);
 151 }
 152
 153 static void action_wakeup_interruptible(struct nvhost_waitlist *waiter)
 154 {
 155         wait_queue_head_t *wq = waiter->data;
 156
 157         wake_up_interruptible(wq);
 158 }
 159
 160 typedef void (*action_handler)(struct nvhost_waitlist *waiter);
 161
 162 static action_handler action_handlers[NVHOST_INTR_ACTION_COUNT] = {
 163         action_submit_complete,
 164         action_ctxsave,
 165         action_wakeup,
 166         action_wakeup_interruptible,
 167 };
 168
 169 static void run_handlers(struct list_head completed[NVHOST_INTR_ACTION_COUNT])
 170 {
 171         struct list_head *head = completed;
 172         int i;
 173
 174         for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i, ++head) {
 175                 action_handler handler = action_handlers[i];
 176                 struct nvhost_waitlist *waiter, *next;
 177
 178                 list_for_each_entry_safe(waiter, next, head, list) {
 179                         list_del(&waiter->list);
 180                         handler(waiter);
 181                         atomic_set(&waiter->state, WLS_HANDLED);
 182                         smp_wmb();
 183                         kref_put(&waiter->refcount, waiter_release);
 184                 }
 185         }
 186 }
 187
 188
 189 /*** Interrupt service functions ***/
 190
 191 /**
 192  * Host1x intterrupt service function
 193  * Handles read / write failures
 194  */
 195 static irqreturn_t host1x_isr(int irq, void *dev_id)
 196 {
 197         struct nvhost_intr *intr = dev_id;
 198         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
 199         u32 stat;
 200         u32 ext_stat;
 201         u32 addr;
 202
 203         stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS);
 204         ext_stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
 205
 206         if (nvhost_sync_hintstatus_ext_ip_read_int(ext_stat)) {
 207                 addr = readl(sync_regs + HOST1X_SYNC_IP_READ_TIMEOUT_ADDR);
 208                 pr_err("Host read timeout at address %x\n", addr);
 209         }
 210
 211         if (nvhost_sync_hintstatus_ext_ip_write_int(ext_stat)) {
 212                 addr = readl(sync_regs + HOST1X_SYNC_IP_WRITE_TIMEOUT_ADDR);
 213                 pr_err("Host write timeout at address %x\n", addr);
 214         }
 215
 216         writel(ext_stat, sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
 217         writel(stat, sync_regs + HOST1X_SYNC_HINTSTATUS);
 218
 219         return IRQ_HANDLED;
 220 }
 221
 222 /**
 223  * Sync point threshold interrupt service function
 224  * Handles sync point threshold triggers, in interrupt context
 225  */
 226 static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
 227 {
 228         struct nvhost_intr_syncpt *syncpt = dev_id;
 229         unsigned int id = syncpt->id;
 230         struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
 231                                                 syncpt[id]);
 232         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
 233
 234         writel(BIT(id),
 235                 sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE);
 236         writel(BIT(id),
 237                 sync_regs + HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS);
 238
 239         return IRQ_WAKE_THREAD;
 240 }
 241
 242
 243 /**
 244  * Sync point threshold interrupt service thread function
 245  * Handles sync point threshold triggers, in thread context
 246  */
 247 static irqreturn_t syncpt_thresh_fn(int irq, void *dev_id)
 248 {
 249         struct nvhost_intr_syncpt *syncpt = dev_id;
 250         unsigned int id = syncpt->id;
 251         struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
 252                                                 syncpt[id]);
 253         struct nvhost_master *dev = intr_to_dev(intr);
 254         void __iomem *sync_regs = dev->sync_aperture;
 255
 256         struct list_head completed[NVHOST_INTR_ACTION_COUNT];
 257         u32 sync;
 258         unsigned int i;
 259
 260         for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i)
 261                 INIT_LIST_HEAD(completed + i);
 262
 263         sync = nvhost_syncpt_update_min(&dev->syncpt, id);
 264
 265         spin_lock(&syncpt->lock);
 266
 267         remove_completed_waiters(&syncpt->wait_head, sync, completed);
 268
 269         if (!list_empty(&syncpt->wait_head)) {
 270                 u32 thresh = list_first_entry(&syncpt->wait_head,
 271                                         struct nvhost_waitlist, list)->thresh;
 272
 273                 set_syncpt_threshold(sync_regs, id, thresh);
 274                 enable_syncpt_interrupt(sync_regs, id);
 275         }
 276
 277         spin_unlock(&syncpt->lock);
 278
 279         run_handlers(completed);
 280
 281         return IRQ_HANDLED;
 282 }
 283
 284 /*
 285  * lazily request a syncpt's irq
 286  */
 287 static int request_syncpt_irq(struct nvhost_intr_syncpt *syncpt)
 288 {
 289         static DEFINE_MUTEX(mutex);
 290         int err;
 291
 292         mutex_lock(&mutex);
 293         if (!syncpt->irq_requested) {
 294                 err = request_threaded_irq(syncpt->irq,
 295                                         syncpt_thresh_isr, syncpt_thresh_fn,
 296                                         0, syncpt->thresh_irq_name, syncpt);
 297                 if (!err)
 298                         syncpt->irq_requested = 1;
 299         }
 300         mutex_unlock(&mutex);
 301         return err;
 302 }
 303
 304
 305 /*** Main API ***/
 306
 307 int nvhost_intr_add_action(struct nvhost_intr *intr, u32 id, u32 thresh,
 308                         enum nvhost_intr_action action, void *data,
 309                         void **ref)
 310 {
 311         struct nvhost_waitlist *waiter;
 312         struct nvhost_intr_syncpt *syncpt;
 313         void __iomem *sync_regs;
 314         int queue_was_empty;
 315         int err;
 316
 317         /* create and initialize a new waiter */
 318         waiter = kmalloc(sizeof(*waiter), GFP_KERNEL);
 319         if (!waiter)
 320                 return -ENOMEM;
 321         INIT_LIST_HEAD(&waiter->list);
 322         kref_init(&waiter->refcount);
 323         if (ref)
 324                 kref_get(&waiter->refcount);
 325         waiter->thresh = thresh;
 326         waiter->action = action;
 327         atomic_set(&waiter->state, WLS_PENDING);
 328         waiter->data = data;
 329         waiter->count = 1;
 330
 331         BUG_ON(id >= NV_HOST1X_SYNCPT_NB_PTS);
 332         syncpt = intr->syncpt + id;
 333         sync_regs = intr_to_dev(intr)->sync_aperture;
 334
 335         spin_lock(&syncpt->lock);
 336
 337         /* lazily request irq for this sync point */
 338         if (!syncpt->irq_requested) {
 339                 spin_unlock(&syncpt->lock);
 340
 341                 err = request_syncpt_irq(syncpt);
 342                 if (err) {
 343                         kfree(waiter);
 344                         return err;
 345                 }
 346
 347                 spin_lock(&syncpt->lock);
 348         }
 349
 350         queue_was_empty = list_empty(&syncpt->wait_head);
 351
 352         if (add_waiter_to_queue(waiter, &syncpt->wait_head)) {
 353                 /* added at head of list - new threshold value */
 354                 set_syncpt_threshold(sync_regs, id, thresh);
 355
 356                 /* added as first waiter - enable interrupt */
 357                 if (queue_was_empty)
 358                         enable_syncpt_interrupt(sync_regs, id);
 359         }
 360
 361         spin_unlock(&syncpt->lock);
 362
 363         if (ref)
 364                 *ref = waiter;
 365         return 0;
 366 }
 367
 368 void nvhost_intr_put_ref(struct nvhost_intr *intr, void *ref)
 369 {
 370         struct nvhost_waitlist *waiter = ref;
 371
 372         while (atomic_cmpxchg(&waiter->state,
 373                                 WLS_PENDING, WLS_CANCELLED) == WLS_REMOVED)
 374                 schedule();
 375
 376         kref_put(&waiter->refcount, waiter_release);
 377 }
 378
 379
 380 /*** Init & shutdown ***/
 381
 382 int nvhost_intr_init(struct nvhost_intr *intr, u32 irq_gen, u32 irq_sync)
 383 {
 384         unsigned int id;
 385         struct nvhost_intr_syncpt *syncpt;
 386         int err;
 387
 388         err = request_irq(irq_gen, host1x_isr, 0, "host_status", intr);
 389         if (err)
 390                 goto fail;
 391         intr->host1x_irq = irq_gen;
 392         intr->host1x_isr_started = true;
 393
 394         for (id = 0, syncpt = intr->syncpt;
 395              id < NV_HOST1X_SYNCPT_NB_PTS;
 396              ++id, ++syncpt) {
 397                 syncpt->id = id;
 398                 syncpt->irq = irq_sync + id;
 399                 syncpt->irq_requested = 0;
 400                 spin_lock_init(&syncpt->lock);
 401                 INIT_LIST_HEAD(&syncpt->wait_head);
 402                 snprintf(syncpt->thresh_irq_name,
 403                          sizeof(syncpt->thresh_irq_name),
 404                          "%s", nvhost_syncpt_name(id));
 405         }
 406
 407         return 0;
 408
 409 fail:
 410         nvhost_intr_deinit(intr);
 411         return err;
 412 }
 413
 414 void nvhost_intr_deinit(struct nvhost_intr *intr)
 415 {
 416         unsigned int id;
 417         struct nvhost_intr_syncpt *syncpt;
 418
 419         for (id = 0, syncpt = intr->syncpt;
 420              id < NV_HOST1X_SYNCPT_NB_PTS;
 421              ++id, ++syncpt)
 422                 if (syncpt->irq_requested)
 423                         free_irq(syncpt->irq, syncpt);
 424
 425         if (intr->host1x_isr_started) {
 426                 free_irq(intr->host1x_irq, intr);
 427                 intr->host1x_isr_started = false;
 428         }
 429 }
 430
 431 void nvhost_intr_configure (struct nvhost_intr *intr, u32 hz)
 432 {
 433         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
 434
 435         // write microsecond clock register
 436         writel((hz + 1000000 - 1)/1000000, sync_regs + HOST1X_SYNC_USEC_CLK);
 437
 438         /* disable the ip_busy_timeout. this prevents write drops, etc.
 439          * there's no real way to recover from a hung client anyway.
 440          */
 441         writel(0, sync_regs + HOST1X_SYNC_IP_BUSY_TIMEOUT);
 442
 443         /* increase the auto-ack timout to the maximum value. 2d will hang
 444          * otherwise on ap20.
 445          */
 446         writel(0xff, sync_regs + HOST1X_SYNC_CTXSW_TIMEOUT_CFG);
 447
 448         /* disable interrupts for both cpu's */
 449         writel(0, sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_MASK_0);
 450         writel(0, sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_MASK_1);
 451
 452         /* masking all of the interrupts actually means "enable" */
 453         writel(BIT(0), sync_regs + HOST1X_SYNC_INTMASK);
 454
 455         /* enable HOST_INT_C0MASK */
 456         writel(BIT(0), sync_regs + HOST1X_SYNC_INTC0MASK);
 457
 458         /* enable HINTMASK_EXT */
 459         writel(BIT(31), sync_regs + HOST1X_SYNC_HINTMASK);
 460
 461         /* enable IP_READ_INT and IP_WRITE_INT */
 462         writel(BIT(30) | BIT(31), sync_regs + HOST1X_SYNC_HINTMASK_EXT);
 463 }