2 * Copyright (C) 2007 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include <linux/ktime.h>
31 #include <linux/hrtimer.h>
33 #include "nouveau_drv.h"
34 #include "nouveau_ramht.h"
35 #include "nouveau_dma.h"
37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
40 struct nouveau_fence {
41 struct nouveau_channel *channel;
43 struct list_head entry;
48 void (*work)(void *priv, bool signalled);
52 struct nouveau_semaphore {
54 struct drm_device *dev;
55 struct drm_mm_node *mem;
58 static inline struct nouveau_fence *
59 nouveau_fence(void *sync_obj)
61 return (struct nouveau_fence *)sync_obj;
65 nouveau_fence_del(struct kref *ref)
67 struct nouveau_fence *fence =
68 container_of(ref, struct nouveau_fence, refcount);
70 nouveau_channel_ref(NULL, &fence->channel);
75 nouveau_fence_update(struct nouveau_channel *chan)
77 struct drm_device *dev = chan->dev;
78 struct nouveau_fence *tmp, *fence;
81 spin_lock(&chan->fence.lock);
83 /* Fetch the last sequence if the channel is still up and running */
84 if (likely(!list_empty(&chan->fence.pending))) {
86 sequence = nvchan_rd32(chan, 0x48);
88 sequence = atomic_read(&chan->fence.last_sequence_irq);
90 if (chan->fence.sequence_ack == sequence)
92 chan->fence.sequence_ack = sequence;
95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
96 sequence = fence->sequence;
97 fence->signalled = true;
98 list_del(&fence->entry);
100 if (unlikely(fence->work))
101 fence->work(fence->priv, true);
103 kref_put(&fence->refcount, nouveau_fence_del);
105 if (sequence == chan->fence.sequence_ack)
109 spin_unlock(&chan->fence.lock);
113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
116 struct nouveau_fence *fence;
119 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
122 kref_init(&fence->refcount);
123 nouveau_channel_ref(chan, &fence->channel);
126 ret = nouveau_fence_emit(fence);
129 nouveau_fence_unref(&fence);
134 struct nouveau_channel *
135 nouveau_fence_channel(struct nouveau_fence *fence)
137 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
141 nouveau_fence_emit(struct nouveau_fence *fence)
143 struct nouveau_channel *chan = fence->channel;
144 struct drm_device *dev = chan->dev;
145 struct drm_nouveau_private *dev_priv = dev->dev_private;
148 ret = RING_SPACE(chan, 2);
152 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
153 nouveau_fence_update(chan);
155 BUG_ON(chan->fence.sequence ==
156 chan->fence.sequence_ack - 1);
159 fence->sequence = ++chan->fence.sequence;
161 kref_get(&fence->refcount);
162 spin_lock(&chan->fence.lock);
163 list_add_tail(&fence->entry, &chan->fence.pending);
164 spin_unlock(&chan->fence.lock);
166 if (USE_REFCNT(dev)) {
167 if (dev_priv->card_type < NV_C0)
168 BEGIN_RING(chan, NvSubSw, 0x0050, 1);
170 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
172 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
174 OUT_RING (chan, fence->sequence);
181 nouveau_fence_work(struct nouveau_fence *fence,
182 void (*work)(void *priv, bool signalled),
187 spin_lock(&fence->channel->fence.lock);
189 if (fence->signalled) {
196 spin_unlock(&fence->channel->fence.lock);
200 __nouveau_fence_unref(void **sync_obj)
202 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
205 kref_put(&fence->refcount, nouveau_fence_del);
210 __nouveau_fence_ref(void *sync_obj)
212 struct nouveau_fence *fence = nouveau_fence(sync_obj);
214 kref_get(&fence->refcount);
219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
221 struct nouveau_fence *fence = nouveau_fence(sync_obj);
222 struct nouveau_channel *chan = fence->channel;
224 if (fence->signalled)
227 nouveau_fence_update(chan);
228 return fence->signalled;
232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
234 unsigned long timeout = jiffies + (3 * DRM_HZ);
235 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
240 if (__nouveau_fence_signalled(sync_obj, sync_arg))
243 if (time_after_eq(jiffies, timeout)) {
248 __set_current_state(intr ? TASK_INTERRUPTIBLE
249 : TASK_UNINTERRUPTIBLE);
251 t = ktime_set(0, sleep_time);
252 schedule_hrtimeout(&t, HRTIMER_MODE_REL);
254 if (sleep_time > NSEC_PER_MSEC)
255 sleep_time = NSEC_PER_MSEC;
258 if (intr && signal_pending(current)) {
264 __set_current_state(TASK_RUNNING);
269 static struct nouveau_semaphore *
270 semaphore_alloc(struct drm_device *dev)
272 struct drm_nouveau_private *dev_priv = dev->dev_private;
273 struct nouveau_semaphore *sema;
274 int size = (dev_priv->chipset < 0x84) ? 4 : 16;
280 sema = kmalloc(sizeof(*sema), GFP_KERNEL);
284 ret = drm_mm_pre_get(&dev_priv->fence.heap);
288 spin_lock(&dev_priv->fence.lock);
289 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
291 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
292 spin_unlock(&dev_priv->fence.lock);
297 kref_init(&sema->ref);
299 for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
300 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
309 semaphore_free(struct kref *ref)
311 struct nouveau_semaphore *sema =
312 container_of(ref, struct nouveau_semaphore, ref);
313 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
315 spin_lock(&dev_priv->fence.lock);
316 drm_mm_put_block(sema->mem);
317 spin_unlock(&dev_priv->fence.lock);
323 semaphore_work(void *priv, bool signalled)
325 struct nouveau_semaphore *sema = priv;
326 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
328 if (unlikely(!signalled))
329 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
331 kref_put(&sema->ref, semaphore_free);
335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
337 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
338 struct nouveau_fence *fence = NULL;
341 if (dev_priv->chipset < 0x84) {
342 ret = RING_SPACE(chan, 4);
346 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
347 OUT_RING (chan, NvSema);
348 OUT_RING (chan, sema->mem->start);
351 if (dev_priv->chipset < 0xc0) {
352 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
353 u64 offset = vma->offset + sema->mem->start;
355 ret = RING_SPACE(chan, 7);
359 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
360 OUT_RING (chan, chan->vram_handle);
361 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
362 OUT_RING (chan, upper_32_bits(offset));
363 OUT_RING (chan, lower_32_bits(offset));
365 OUT_RING (chan, 1); /* ACQUIRE_EQ */
367 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
368 u64 offset = vma->offset + sema->mem->start;
370 ret = RING_SPACE(chan, 5);
374 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
375 OUT_RING (chan, upper_32_bits(offset));
376 OUT_RING (chan, lower_32_bits(offset));
378 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */
381 /* Delay semaphore destruction until its work is done */
382 ret = nouveau_fence_new(chan, &fence, true);
386 kref_get(&sema->ref);
387 nouveau_fence_work(fence, semaphore_work, sema);
388 nouveau_fence_unref(&fence);
393 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
395 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
396 struct nouveau_fence *fence = NULL;
399 if (dev_priv->chipset < 0x84) {
400 ret = RING_SPACE(chan, 5);
404 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
405 OUT_RING (chan, NvSema);
406 OUT_RING (chan, sema->mem->start);
407 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
410 if (dev_priv->chipset < 0xc0) {
411 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
412 u64 offset = vma->offset + sema->mem->start;
414 ret = RING_SPACE(chan, 7);
418 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
419 OUT_RING (chan, chan->vram_handle);
420 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
421 OUT_RING (chan, upper_32_bits(offset));
422 OUT_RING (chan, lower_32_bits(offset));
424 OUT_RING (chan, 2); /* RELEASE */
426 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
427 u64 offset = vma->offset + sema->mem->start;
429 ret = RING_SPACE(chan, 5);
433 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
434 OUT_RING (chan, upper_32_bits(offset));
435 OUT_RING (chan, lower_32_bits(offset));
437 OUT_RING (chan, 0x1002); /* RELEASE */
440 /* Delay semaphore destruction until its work is done */
441 ret = nouveau_fence_new(chan, &fence, true);
445 kref_get(&sema->ref);
446 nouveau_fence_work(fence, semaphore_work, sema);
447 nouveau_fence_unref(&fence);
452 nouveau_fence_sync(struct nouveau_fence *fence,
453 struct nouveau_channel *wchan)
455 struct nouveau_channel *chan = nouveau_fence_channel(fence);
456 struct drm_device *dev = wchan->dev;
457 struct nouveau_semaphore *sema;
460 if (likely(!chan || chan == wchan ||
461 nouveau_fence_signalled(fence)))
464 sema = semaphore_alloc(dev);
466 /* Early card or broken userspace, fall back to
468 ret = nouveau_fence_wait(fence, true, false);
472 /* try to take chan's mutex, if we can't take it right away
473 * we have to fallback to software sync to prevent locking
476 if (!mutex_trylock(&chan->mutex)) {
477 ret = nouveau_fence_wait(fence, true, false);
481 /* Make wchan wait until it gets signalled */
482 ret = semaphore_acquire(wchan, sema);
486 /* Signal the semaphore from chan */
487 ret = semaphore_release(chan, sema);
490 mutex_unlock(&chan->mutex);
492 kref_put(&sema->ref, semaphore_free);
495 nouveau_channel_put_unlocked(&chan);
500 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
506 nouveau_fence_channel_init(struct nouveau_channel *chan)
508 struct drm_device *dev = chan->dev;
509 struct drm_nouveau_private *dev_priv = dev->dev_private;
510 struct nouveau_gpuobj *obj = NULL;
513 if (dev_priv->card_type < NV_C0) {
514 /* Create an NV_SW object for various sync purposes */
515 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
519 ret = RING_SPACE(chan, 2);
523 BEGIN_RING(chan, NvSubSw, 0, 1);
524 OUT_RING (chan, NvSw);
528 /* Setup area of memory shared between all channels for x-chan sync */
529 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
530 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
532 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
533 mem->start << PAGE_SHIFT,
534 mem->size, NV_MEM_ACCESS_RW,
535 NV_MEM_TARGET_VRAM, &obj);
539 ret = nouveau_ramht_insert(chan, NvSema, obj);
540 nouveau_gpuobj_ref(NULL, &obj);
545 atomic_set(&chan->fence.last_sequence_irq, 0);
550 nouveau_fence_channel_fini(struct nouveau_channel *chan)
552 struct nouveau_fence *tmp, *fence;
554 spin_lock(&chan->fence.lock);
556 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
557 fence->signalled = true;
558 list_del(&fence->entry);
560 if (unlikely(fence->work))
561 fence->work(fence->priv, false);
563 kref_put(&fence->refcount, nouveau_fence_del);
566 spin_unlock(&chan->fence.lock);
570 nouveau_fence_init(struct drm_device *dev)
572 struct drm_nouveau_private *dev_priv = dev->dev_private;
573 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
576 /* Create a shared VRAM heap for cross-channel sync. */
578 ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
579 0, 0, &dev_priv->fence.bo);
583 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
587 ret = nouveau_bo_map(dev_priv->fence.bo);
591 ret = drm_mm_init(&dev_priv->fence.heap, 0,
592 dev_priv->fence.bo->bo.mem.size);
596 spin_lock_init(&dev_priv->fence.lock);
601 nouveau_bo_unmap(dev_priv->fence.bo);
602 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
607 nouveau_fence_fini(struct drm_device *dev)
609 struct drm_nouveau_private *dev_priv = dev->dev_private;
612 drm_mm_takedown(&dev_priv->fence.heap);
613 nouveau_bo_unmap(dev_priv->fence.bo);
614 nouveau_bo_unpin(dev_priv->fence.bo);
615 nouveau_bo_ref(NULL, &dev_priv->fence.bo);