1 /*****************************************************************************
2 * Copyright 2004 - 2008 Broadcom Corporation. All rights reserved.
4 * Unless you and Broadcom execute a separate written software license
5 * agreement governing use of this software, this software is licensed to you
6 * under the terms of the GNU General Public License version 2, available at
7 * http://www.broadcom.com/licenses/GPLv2.php (the "GPL").
9 * Notwithstanding the above, under no circumstances may you combine this
10 * software in any way with any other Broadcom software provided under a
11 * license other than the GPL, without Broadcom's express prior written
13 *****************************************************************************/
15 /****************************************************************************/
19 * @brief Implements the DMA interface.
21 /****************************************************************************/
23 /* ---- Include Files ---------------------------------------------------- */
25 #include <linux/module.h>
26 #include <linux/device.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/interrupt.h>
29 #include <linux/sched.h>
30 #include <linux/irqreturn.h>
31 #include <linux/proc_fs.h>
32 #include <linux/slab.h>
34 #include <mach/timer.h>
37 #include <linux/pfn.h>
38 #include <linux/atomic.h>
41 /* I don't quite understand why dc4 fails when this is set to 1 and DMA is enabled */
42 /* especially since dc4 doesn't use kmalloc'd memory. */
44 #define ALLOW_MAP_OF_KMALLOC_MEMORY 0
46 /* ---- Public Variables ------------------------------------------------- */
48 /* ---- Private Constants and Types -------------------------------------- */
50 #define MAKE_HANDLE(controllerIdx, channelIdx) (((controllerIdx) << 4) | (channelIdx))
52 #define CONTROLLER_FROM_HANDLE(handle) (((handle) >> 4) & 0x0f)
53 #define CHANNEL_FROM_HANDLE(handle) ((handle) & 0x0f)
55 #define DMA_MAP_DEBUG 0
58 # define DMA_MAP_PRINT(fmt, args...) printk("%s: " fmt, __func__, ## args)
60 # define DMA_MAP_PRINT(fmt, args...)
63 /* ---- Private Variables ------------------------------------------------ */
65 static DMA_Global_t gDMA;
66 static struct proc_dir_entry *gDmaDir;
68 static atomic_t gDmaStatMemTypeKmalloc = ATOMIC_INIT(0);
69 static atomic_t gDmaStatMemTypeVmalloc = ATOMIC_INIT(0);
70 static atomic_t gDmaStatMemTypeUser = ATOMIC_INIT(0);
71 static atomic_t gDmaStatMemTypeCoherent = ATOMIC_INIT(0);
73 #include "dma_device.c"
75 /* ---- Private Function Prototypes -------------------------------------- */
77 /* ---- Functions ------------------------------------------------------- */
79 /****************************************************************************/
81 * Displays information for /proc/dma/mem-type
83 /****************************************************************************/
85 static int dma_proc_read_mem_type(char *buf, char **start, off_t offset,
86 int count, int *eof, void *data)
90 len += sprintf(buf + len, "dma_map_mem statistics\n");
92 sprintf(buf + len, "coherent: %d\n",
93 atomic_read(&gDmaStatMemTypeCoherent));
95 sprintf(buf + len, "kmalloc: %d\n",
96 atomic_read(&gDmaStatMemTypeKmalloc));
98 sprintf(buf + len, "vmalloc: %d\n",
99 atomic_read(&gDmaStatMemTypeVmalloc));
101 sprintf(buf + len, "user: %d\n",
102 atomic_read(&gDmaStatMemTypeUser));
107 /****************************************************************************/
109 * Displays information for /proc/dma/channels
111 /****************************************************************************/
113 static int dma_proc_read_channels(char *buf, char **start, off_t offset,
114 int count, int *eof, void *data)
118 int limit = count - 200;
120 DMA_Channel_t *channel;
122 if (down_interruptible(&gDMA.lock) < 0) {
126 for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
128 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
135 &gDMA.controller[controllerIdx].channel[channelIdx];
138 sprintf(buf + len, "%d:%d ", controllerIdx,
141 if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
144 sprintf(buf + len, "Dedicated for %s ",
145 DMA_gDeviceAttribute[channel->
148 len += sprintf(buf + len, "Shared ");
151 if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) != 0) {
152 len += sprintf(buf + len, "No ISR ");
155 if ((channel->flags & DMA_CHANNEL_FLAG_LARGE_FIFO) != 0) {
156 len += sprintf(buf + len, "Fifo: 128 ");
158 len += sprintf(buf + len, "Fifo: 64 ");
161 if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
163 sprintf(buf + len, "InUse by %s",
164 DMA_gDeviceAttribute[channel->
166 #if (DMA_DEBUG_TRACK_RESERVATION)
168 sprintf(buf + len, " (%s:%d)",
173 len += sprintf(buf + len, "Avail ");
176 if (channel->lastDevType != DMA_DEVICE_NONE) {
178 sprintf(buf + len, "Last use: %s ",
179 DMA_gDeviceAttribute[channel->
184 len += sprintf(buf + len, "\n");
193 /****************************************************************************/
195 * Displays information for /proc/dma/devices
197 /****************************************************************************/
199 static int dma_proc_read_devices(char *buf, char **start, off_t offset,
200 int count, int *eof, void *data)
202 int limit = count - 200;
206 if (down_interruptible(&gDMA.lock) < 0) {
210 for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
211 DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];
213 if (devAttr->name == NULL) {
221 len += sprintf(buf + len, "%-12s ", devAttr->name);
223 if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
225 sprintf(buf + len, "Dedicated %d:%d ",
226 devAttr->dedicatedController,
227 devAttr->dedicatedChannel);
229 len += sprintf(buf + len, "Shared DMA:");
230 if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA0) != 0) {
231 len += sprintf(buf + len, "0");
233 if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA1) != 0) {
234 len += sprintf(buf + len, "1");
236 len += sprintf(buf + len, " ");
238 if ((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0) {
239 len += sprintf(buf + len, "NoISR ");
241 if ((devAttr->flags & DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO) != 0) {
242 len += sprintf(buf + len, "Allow-128 ");
247 "Xfer #: %Lu Ticks: %Lu Bytes: %Lu DescLen: %u\n",
248 devAttr->numTransfers, devAttr->transferTicks,
249 devAttr->transferBytes,
250 devAttr->ring.bytesAllocated);
260 /****************************************************************************/
262 * Determines if a DMA_Device_t is "valid".
265 * TRUE - dma device is valid
266 * FALSE - dma device isn't valid
268 /****************************************************************************/
270 static inline int IsDeviceValid(DMA_Device_t device)
272 return (device >= 0) && (device < DMA_NUM_DEVICE_ENTRIES);
275 /****************************************************************************/
277 * Translates a DMA handle into a pointer to a channel.
280 * non-NULL - pointer to DMA_Channel_t
281 * NULL - DMA Handle was invalid
283 /****************************************************************************/
285 static inline DMA_Channel_t *HandleToChannel(DMA_Handle_t handle)
290 controllerIdx = CONTROLLER_FROM_HANDLE(handle);
291 channelIdx = CHANNEL_FROM_HANDLE(handle);
293 if ((controllerIdx > DMA_NUM_CONTROLLERS)
294 || (channelIdx > DMA_NUM_CHANNELS)) {
297 return &gDMA.controller[controllerIdx].channel[channelIdx];
300 /****************************************************************************/
302 * Interrupt handler which is called to process DMA interrupts.
304 /****************************************************************************/
306 static irqreturn_t dma_interrupt_handler(int irq, void *dev_id)
308 DMA_Channel_t *channel;
309 DMA_DeviceAttribute_t *devAttr;
312 channel = (DMA_Channel_t *) dev_id;
314 /* Figure out why we were called, and knock down the interrupt */
316 irqStatus = dmacHw_getInterruptStatus(channel->dmacHwHandle);
317 dmacHw_clearInterrupt(channel->dmacHwHandle);
319 if ((channel->devType < 0)
320 || (channel->devType > DMA_NUM_DEVICE_ENTRIES)) {
321 printk(KERN_ERR "dma_interrupt_handler: Invalid devType: %d\n",
325 devAttr = &DMA_gDeviceAttribute[channel->devType];
329 if ((irqStatus & dmacHw_INTERRUPT_STATUS_TRANS) != 0) {
330 devAttr->transferTicks +=
331 (timer_get_tick_count() - devAttr->transferStartTime);
334 if ((irqStatus & dmacHw_INTERRUPT_STATUS_ERROR) != 0) {
336 "dma_interrupt_handler: devType :%d DMA error (%s)\n",
337 channel->devType, devAttr->name);
339 devAttr->numTransfers++;
340 devAttr->transferBytes += devAttr->numBytes;
343 /* Call any installed handler */
345 if (devAttr->devHandler != NULL) {
346 devAttr->devHandler(channel->devType, irqStatus,
353 /****************************************************************************/
355 * Allocates memory to hold a descriptor ring. The descriptor ring then
356 * needs to be populated by making one or more calls to
357 * dna_add_descriptors.
359 * The returned descriptor ring will be automatically initialized.
362 * 0 Descriptor ring was allocated successfully
363 * -EINVAL Invalid parameters passed in
364 * -ENOMEM Unable to allocate memory for the desired number of descriptors.
366 /****************************************************************************/
368 int dma_alloc_descriptor_ring(DMA_DescriptorRing_t *ring, /* Descriptor ring to populate */
369 int numDescriptors /* Number of descriptors that need to be allocated. */
371 size_t bytesToAlloc = dmacHw_descriptorLen(numDescriptors);
373 if ((ring == NULL) || (numDescriptors <= 0)) {
378 ring->descriptorsAllocated = 0;
379 ring->bytesAllocated = 0;
381 ring->virtAddr = dma_alloc_writecombine(NULL,
385 if (ring->virtAddr == NULL) {
389 ring->bytesAllocated = bytesToAlloc;
390 ring->descriptorsAllocated = numDescriptors;
392 return dma_init_descriptor_ring(ring, numDescriptors);
395 EXPORT_SYMBOL(dma_alloc_descriptor_ring);
397 /****************************************************************************/
399 * Releases the memory which was previously allocated for a descriptor ring.
401 /****************************************************************************/
403 void dma_free_descriptor_ring(DMA_DescriptorRing_t *ring /* Descriptor to release */
405 if (ring->virtAddr != NULL) {
406 dma_free_writecombine(NULL,
407 ring->bytesAllocated,
408 ring->virtAddr, ring->physAddr);
411 ring->bytesAllocated = 0;
412 ring->descriptorsAllocated = 0;
413 ring->virtAddr = NULL;
417 EXPORT_SYMBOL(dma_free_descriptor_ring);
419 /****************************************************************************/
421 * Initializes a descriptor ring, so that descriptors can be added to it.
422 * Once a descriptor ring has been allocated, it may be reinitialized for
423 * use with additional/different regions of memory.
425 * Note that if 7 descriptors are allocated, it's perfectly acceptable to
426 * initialize the ring with a smaller number of descriptors. The amount
427 * of memory allocated for the descriptor ring will not be reduced, and
428 * the descriptor ring may be reinitialized later
431 * 0 Descriptor ring was initialized successfully
432 * -ENOMEM The descriptor which was passed in has insufficient space
433 * to hold the desired number of descriptors.
435 /****************************************************************************/
437 int dma_init_descriptor_ring(DMA_DescriptorRing_t *ring, /* Descriptor ring to initialize */
438 int numDescriptors /* Number of descriptors to initialize. */
440 if (ring->virtAddr == NULL) {
443 if (dmacHw_initDescriptor(ring->virtAddr,
445 ring->bytesAllocated, numDescriptors) < 0) {
447 "dma_init_descriptor_ring: dmacHw_initDescriptor failed\n");
454 EXPORT_SYMBOL(dma_init_descriptor_ring);
456 /****************************************************************************/
458 * Determines the number of descriptors which would be required for a
459 * transfer of the indicated memory region.
461 * This function also needs to know which DMA device this transfer will
462 * be destined for, so that the appropriate DMA configuration can be retrieved.
463 * DMA parameters such as transfer width, and whether this is a memory-to-memory
464 * or memory-to-peripheral, etc can all affect the actual number of descriptors
468 * > 0 Returns the number of descriptors required for the indicated transfer
469 * -ENODEV - Device handed in is invalid.
470 * -EINVAL Invalid parameters
471 * -ENOMEM Memory exhausted
473 /****************************************************************************/
475 int dma_calculate_descriptor_count(DMA_Device_t device, /* DMA Device that this will be associated with */
476 dma_addr_t srcData, /* Place to get data to write to device */
477 dma_addr_t dstData, /* Pointer to device data address */
478 size_t numBytes /* Number of bytes to transfer to the device */
481 DMA_DeviceAttribute_t *devAttr;
483 if (!IsDeviceValid(device)) {
486 devAttr = &DMA_gDeviceAttribute[device];
488 numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
492 if (numDescriptors < 0) {
494 "dma_calculate_descriptor_count: dmacHw_calculateDescriptorCount failed\n");
498 return numDescriptors;
501 EXPORT_SYMBOL(dma_calculate_descriptor_count);
503 /****************************************************************************/
505 * Adds a region of memory to the descriptor ring. Note that it may take
506 * multiple descriptors for each region of memory. It is the callers
507 * responsibility to allocate a sufficiently large descriptor ring.
510 * 0 Descriptors were added successfully
511 * -ENODEV Device handed in is invalid.
512 * -EINVAL Invalid parameters
513 * -ENOMEM Memory exhausted
515 /****************************************************************************/
517 int dma_add_descriptors(DMA_DescriptorRing_t *ring, /* Descriptor ring to add descriptors to */
518 DMA_Device_t device, /* DMA Device that descriptors are for */
519 dma_addr_t srcData, /* Place to get data (memory or device) */
520 dma_addr_t dstData, /* Place to put data (memory or device) */
521 size_t numBytes /* Number of bytes to transfer to the device */
524 DMA_DeviceAttribute_t *devAttr;
526 if (!IsDeviceValid(device)) {
529 devAttr = &DMA_gDeviceAttribute[device];
531 rc = dmacHw_setDataDescriptor(&devAttr->config,
534 (void *)dstData, numBytes);
537 "dma_add_descriptors: dmacHw_setDataDescriptor failed with code: %d\n",
545 EXPORT_SYMBOL(dma_add_descriptors);
547 /****************************************************************************/
549 * Sets the descriptor ring associated with a device.
551 * Once set, the descriptor ring will be associated with the device, even
552 * across channel request/free calls. Passing in a NULL descriptor ring
553 * will release any descriptor ring currently associated with the device.
555 * Note: If you call dma_transfer, or one of the other dma_alloc_ functions
556 * the descriptor ring may be released and reallocated.
558 * Note: This function will release the descriptor memory for any current
559 * descriptor ring associated with this device.
562 * 0 Descriptors were added successfully
563 * -ENODEV Device handed in is invalid.
565 /****************************************************************************/
567 int dma_set_device_descriptor_ring(DMA_Device_t device, /* Device to update the descriptor ring for. */
568 DMA_DescriptorRing_t *ring /* Descriptor ring to add descriptors to */
570 DMA_DeviceAttribute_t *devAttr;
572 if (!IsDeviceValid(device)) {
575 devAttr = &DMA_gDeviceAttribute[device];
577 /* Free the previously allocated descriptor ring */
579 dma_free_descriptor_ring(&devAttr->ring);
582 /* Copy in the new one */
584 devAttr->ring = *ring;
587 /* Set things up so that if dma_transfer is called then this descriptor */
588 /* ring will get freed. */
590 devAttr->prevSrcData = 0;
591 devAttr->prevDstData = 0;
592 devAttr->prevNumBytes = 0;
597 EXPORT_SYMBOL(dma_set_device_descriptor_ring);
599 /****************************************************************************/
601 * Retrieves the descriptor ring associated with a device.
604 * 0 Descriptors were added successfully
605 * -ENODEV Device handed in is invalid.
607 /****************************************************************************/
609 int dma_get_device_descriptor_ring(DMA_Device_t device, /* Device to retrieve the descriptor ring for. */
610 DMA_DescriptorRing_t *ring /* Place to store retrieved ring */
612 DMA_DeviceAttribute_t *devAttr;
614 memset(ring, 0, sizeof(*ring));
616 if (!IsDeviceValid(device)) {
619 devAttr = &DMA_gDeviceAttribute[device];
621 *ring = devAttr->ring;
626 EXPORT_SYMBOL(dma_get_device_descriptor_ring);
628 /****************************************************************************/
630 * Configures a DMA channel.
633 * >= 0 - Initialization was successful.
635 * -EBUSY - Device is currently being used.
636 * -ENODEV - Device handed in is invalid.
638 /****************************************************************************/
640 static int ConfigChannel(DMA_Handle_t handle)
642 DMA_Channel_t *channel;
643 DMA_DeviceAttribute_t *devAttr;
646 channel = HandleToChannel(handle);
647 if (channel == NULL) {
650 devAttr = &DMA_gDeviceAttribute[channel->devType];
651 controllerIdx = CONTROLLER_FROM_HANDLE(handle);
653 if ((devAttr->flags & DMA_DEVICE_FLAG_PORT_PER_DMAC) != 0) {
654 if (devAttr->config.transferType ==
655 dmacHw_TRANSFER_TYPE_MEM_TO_PERIPHERAL) {
656 devAttr->config.dstPeripheralPort =
657 devAttr->dmacPort[controllerIdx];
658 } else if (devAttr->config.transferType ==
659 dmacHw_TRANSFER_TYPE_PERIPHERAL_TO_MEM) {
660 devAttr->config.srcPeripheralPort =
661 devAttr->dmacPort[controllerIdx];
665 if (dmacHw_configChannel(channel->dmacHwHandle, &devAttr->config) != 0) {
666 printk(KERN_ERR "ConfigChannel: dmacHw_configChannel failed\n");
673 /****************************************************************************/
675 * Initializes all of the data structures associated with the DMA.
677 * >= 0 - Initialization was successful.
679 * -EBUSY - Device is currently being used.
680 * -ENODEV - Device handed in is invalid.
682 /****************************************************************************/
690 DMA_Channel_t *channel;
691 DMA_Handle_t dedicatedHandle;
693 memset(&gDMA, 0, sizeof(gDMA));
695 sema_init(&gDMA.lock, 0);
696 init_waitqueue_head(&gDMA.freeChannelQ);
698 /* Initialize the Hardware */
702 /* Start off by marking all of the DMA channels as shared. */
704 for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
706 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
709 &gDMA.controller[controllerIdx].channel[channelIdx];
712 channel->devType = DMA_DEVICE_NONE;
713 channel->lastDevType = DMA_DEVICE_NONE;
715 #if (DMA_DEBUG_TRACK_RESERVATION)
716 channel->fileName = "";
717 channel->lineNum = 0;
720 channel->dmacHwHandle =
721 dmacHw_getChannelHandle(dmacHw_MAKE_CHANNEL_ID
724 dmacHw_initChannel(channel->dmacHwHandle);
728 /* Record any special attributes that channels may have */
730 gDMA.controller[0].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
731 gDMA.controller[0].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
732 gDMA.controller[1].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
733 gDMA.controller[1].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
735 /* Now walk through and record the dedicated channels. */
737 for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
738 DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];
740 if (((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0)
741 && ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0)) {
743 "DMA Device: %s Can only request NO_ISR for dedicated devices\n",
749 if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
750 /* This is a dedicated device. Mark the channel as being reserved. */
752 if (devAttr->dedicatedController >= DMA_NUM_CONTROLLERS) {
754 "DMA Device: %s DMA Controller %d is out of range\n",
756 devAttr->dedicatedController);
761 if (devAttr->dedicatedChannel >= DMA_NUM_CHANNELS) {
763 "DMA Device: %s DMA Channel %d is out of range\n",
765 devAttr->dedicatedChannel);
771 MAKE_HANDLE(devAttr->dedicatedController,
772 devAttr->dedicatedChannel);
773 channel = HandleToChannel(dedicatedHandle);
775 if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
778 ("DMA Device: %s attempting to use same DMA Controller:Channel (%d:%d) as %s\n",
780 devAttr->dedicatedController,
781 devAttr->dedicatedChannel,
782 DMA_gDeviceAttribute[channel->devType].
788 channel->flags |= DMA_CHANNEL_FLAG_IS_DEDICATED;
789 channel->devType = devIdx;
791 if (devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) {
792 channel->flags |= DMA_CHANNEL_FLAG_NO_ISR;
795 /* For dedicated channels, we can go ahead and configure the DMA channel now */
798 ConfigChannel(dedicatedHandle);
802 /* Go through and register the interrupt handlers */
804 for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
806 for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
809 &gDMA.controller[controllerIdx].channel[channelIdx];
811 if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) == 0) {
812 snprintf(channel->name, sizeof(channel->name),
813 "dma %d:%d %s", controllerIdx,
816 DMA_DEVICE_NONE ? "" :
817 DMA_gDeviceAttribute[channel->devType].
821 request_irq(IRQ_DMA0C0 +
825 dma_interrupt_handler,
826 IRQF_DISABLED, channel->name,
830 "request_irq for IRQ_DMA%dC%d failed\n",
831 controllerIdx, channelIdx);
837 /* Create /proc/dma/channels and /proc/dma/devices */
839 gDmaDir = proc_mkdir("dma", NULL);
841 if (gDmaDir == NULL) {
842 printk(KERN_ERR "Unable to create /proc/dma\n");
844 create_proc_read_entry("channels", 0, gDmaDir,
845 dma_proc_read_channels, NULL);
846 create_proc_read_entry("devices", 0, gDmaDir,
847 dma_proc_read_devices, NULL);
848 create_proc_read_entry("mem-type", 0, gDmaDir,
849 dma_proc_read_mem_type, NULL);
859 /****************************************************************************/
861 * Reserves a channel for use with @a dev. If the device is setup to use
862 * a shared channel, then this function will block until a free channel
866 * >= 0 - A valid DMA Handle.
867 * -EBUSY - Device is currently being used.
868 * -ENODEV - Device handed in is invalid.
870 /****************************************************************************/
872 #if (DMA_DEBUG_TRACK_RESERVATION)
873 DMA_Handle_t dma_request_channel_dbg
874 (DMA_Device_t dev, const char *fileName, int lineNum)
876 DMA_Handle_t dma_request_channel(DMA_Device_t dev)
880 DMA_DeviceAttribute_t *devAttr;
881 DMA_Channel_t *channel;
886 if (down_interruptible(&gDMA.lock) < 0) {
890 if ((dev < 0) || (dev >= DMA_NUM_DEVICE_ENTRIES)) {
894 devAttr = &DMA_gDeviceAttribute[dev];
896 #if (DMA_DEBUG_TRACK_RESERVATION)
900 s = strrchr(fileName, '/');
906 if ((devAttr->flags & DMA_DEVICE_FLAG_IN_USE) != 0) {
907 /* This device has already been requested and not been freed */
909 printk(KERN_ERR "%s: device %s is already requested\n",
910 __func__, devAttr->name);
915 if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
916 /* This device has a dedicated channel. */
919 &gDMA.controller[devAttr->dedicatedController].
920 channel[devAttr->dedicatedChannel];
921 if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
926 channel->flags |= DMA_CHANNEL_FLAG_IN_USE;
927 devAttr->flags |= DMA_DEVICE_FLAG_IN_USE;
929 #if (DMA_DEBUG_TRACK_RESERVATION)
930 channel->fileName = fileName;
931 channel->lineNum = lineNum;
934 MAKE_HANDLE(devAttr->dedicatedController,
935 devAttr->dedicatedChannel);
939 /* This device needs to use one of the shared channels. */
941 handle = DMA_INVALID_HANDLE;
942 while (handle == DMA_INVALID_HANDLE) {
943 /* Scan through the shared channels and see if one is available */
945 for (controllerIdx2 = 0; controllerIdx2 < DMA_NUM_CONTROLLERS;
947 /* Check to see if we should try on controller 1 first. */
949 controllerIdx = controllerIdx2;
951 flags & DMA_DEVICE_FLAG_ALLOC_DMA1_FIRST) != 0) {
952 controllerIdx = 1 - controllerIdx;
955 /* See if the device is available on the controller being tested */
958 flags & (DMA_DEVICE_FLAG_ON_DMA0 << controllerIdx))
961 channelIdx < DMA_NUM_CHANNELS;
964 &gDMA.controller[controllerIdx].
969 DMA_CHANNEL_FLAG_IS_DEDICATED) ==
973 flags & DMA_CHANNEL_FLAG_IN_USE)
977 DMA_CHANNEL_FLAG_LARGE_FIFO)
982 DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO)
984 /* This channel is a large fifo - don't tie it up */
985 /* with devices that we don't want using it. */
991 DMA_CHANNEL_FLAG_IN_USE;
992 channel->devType = dev;
994 DMA_DEVICE_FLAG_IN_USE;
996 #if (DMA_DEBUG_TRACK_RESERVATION)
997 channel->fileName = fileName;
998 channel->lineNum = lineNum;
1001 MAKE_HANDLE(controllerIdx,
1004 /* Now that we've reserved the channel - we can go ahead and configure it */
1006 if (ConfigChannel(handle) != 0) {
1009 "dma_request_channel: ConfigChannel failed\n");
1017 /* No channels are currently available. Let's wait for one to free up. */
1022 prepare_to_wait(&gDMA.freeChannelQ, &wait,
1023 TASK_INTERRUPTIBLE);
1026 finish_wait(&gDMA.freeChannelQ, &wait);
1028 if (signal_pending(current)) {
1029 /* We don't currently hold gDMA.lock, so we return directly */
1031 return -ERESTARTSYS;
1035 if (down_interruptible(&gDMA.lock)) {
1036 return -ERESTARTSYS;
1046 /* Create both _dbg and non _dbg functions for modules. */
1048 #if (DMA_DEBUG_TRACK_RESERVATION)
1049 #undef dma_request_channel
1050 DMA_Handle_t dma_request_channel(DMA_Device_t dev)
1052 return dma_request_channel_dbg(dev, __FILE__, __LINE__);
1055 EXPORT_SYMBOL(dma_request_channel_dbg);
1057 EXPORT_SYMBOL(dma_request_channel);
1059 /****************************************************************************/
1061 * Frees a previously allocated DMA Handle.
1063 /****************************************************************************/
1065 int dma_free_channel(DMA_Handle_t handle /* DMA handle. */
1068 DMA_Channel_t *channel;
1069 DMA_DeviceAttribute_t *devAttr;
1071 if (down_interruptible(&gDMA.lock) < 0) {
1072 return -ERESTARTSYS;
1075 channel = HandleToChannel(handle);
1076 if (channel == NULL) {
1081 devAttr = &DMA_gDeviceAttribute[channel->devType];
1083 if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) == 0) {
1084 channel->lastDevType = channel->devType;
1085 channel->devType = DMA_DEVICE_NONE;
1087 channel->flags &= ~DMA_CHANNEL_FLAG_IN_USE;
1088 devAttr->flags &= ~DMA_DEVICE_FLAG_IN_USE;
1093 wake_up_interruptible(&gDMA.freeChannelQ);
1098 EXPORT_SYMBOL(dma_free_channel);
1100 /****************************************************************************/
1102 * Determines if a given device has been configured as using a shared
1106 * 0 Device uses a dedicated channel
1107 * > zero Device uses a shared channel
1110 /****************************************************************************/
1112 int dma_device_is_channel_shared(DMA_Device_t device /* Device to check. */
1114 DMA_DeviceAttribute_t *devAttr;
1116 if (!IsDeviceValid(device)) {
1119 devAttr = &DMA_gDeviceAttribute[device];
1121 return ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0);
1124 EXPORT_SYMBOL(dma_device_is_channel_shared);
1126 /****************************************************************************/
1128 * Allocates buffers for the descriptors. This is normally done automatically
1129 * but needs to be done explicitly when initiating a dma from interrupt
1133 * 0 Descriptors were allocated successfully
1134 * -EINVAL Invalid device type for this kind of transfer
1135 * (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
1136 * -ENOMEM Memory exhausted
1138 /****************************************************************************/
1140 int dma_alloc_descriptors(DMA_Handle_t handle, /* DMA Handle */
1141 dmacHw_TRANSFER_TYPE_e transferType, /* Type of transfer being performed */
1142 dma_addr_t srcData, /* Place to get data to write to device */
1143 dma_addr_t dstData, /* Pointer to device data address */
1144 size_t numBytes /* Number of bytes to transfer to the device */
1146 DMA_Channel_t *channel;
1147 DMA_DeviceAttribute_t *devAttr;
1149 size_t ringBytesRequired;
1152 channel = HandleToChannel(handle);
1153 if (channel == NULL) {
1157 devAttr = &DMA_gDeviceAttribute[channel->devType];
1159 if (devAttr->config.transferType != transferType) {
1163 /* Figure out how many descriptors we need. */
1165 /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
1166 /* srcData, dstData, numBytes); */
1168 numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
1172 if (numDescriptors < 0) {
1173 printk(KERN_ERR "%s: dmacHw_calculateDescriptorCount failed\n",
1178 /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
1181 ringBytesRequired = dmacHw_descriptorLen(numDescriptors);
1183 /* printk("ringBytesRequired: %d\n", ringBytesRequired); */
1185 if (ringBytesRequired > devAttr->ring.bytesAllocated) {
1186 /* Make sure that this code path is never taken from interrupt context. */
1187 /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
1188 /* allocation needs to have already been done. */
1192 /* Free the old descriptor ring and allocate a new one. */
1194 dma_free_descriptor_ring(&devAttr->ring);
1196 /* And allocate a new one. */
1199 dma_alloc_descriptor_ring(&devAttr->ring,
1203 "%s: dma_alloc_descriptor_ring(%d) failed\n",
1204 __func__, numDescriptors);
1207 /* Setup the descriptor for this transfer */
1209 if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
1210 devAttr->ring.physAddr,
1211 devAttr->ring.bytesAllocated,
1212 numDescriptors) < 0) {
1213 printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n",
1218 /* We've already got enough ring buffer allocated. All we need to do is reset */
1219 /* any control information, just in case the previous DMA was stopped. */
1221 dmacHw_resetDescriptorControl(devAttr->ring.virtAddr);
1224 /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
1225 /* as last time, then we don't need to call setDataDescriptor again. */
1227 if (dmacHw_setDataDescriptor(&devAttr->config,
1228 devAttr->ring.virtAddr,
1230 (void *)dstData, numBytes) < 0) {
1231 printk(KERN_ERR "%s: dmacHw_setDataDescriptor failed\n",
1236 /* Remember the critical information for this transfer so that we can eliminate */
1237 /* another call to dma_alloc_descriptors if the caller reuses the same buffers */
1239 devAttr->prevSrcData = srcData;
1240 devAttr->prevDstData = dstData;
1241 devAttr->prevNumBytes = numBytes;
1246 EXPORT_SYMBOL(dma_alloc_descriptors);
1248 /****************************************************************************/
1250 * Allocates and sets up descriptors for a double buffered circular buffer.
1252 * This is primarily intended to be used for things like the ingress samples
1253 * from a microphone.
1256 * > 0 Number of descriptors actually allocated.
1257 * -EINVAL Invalid device type for this kind of transfer
1258 * (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
1259 * -ENOMEM Memory exhausted
1261 /****************************************************************************/
1263 int dma_alloc_double_dst_descriptors(DMA_Handle_t handle, /* DMA Handle */
1264 dma_addr_t srcData, /* Physical address of source data */
1265 dma_addr_t dstData1, /* Physical address of first destination buffer */
1266 dma_addr_t dstData2, /* Physical address of second destination buffer */
1267 size_t numBytes /* Number of bytes in each destination buffer */
1269 DMA_Channel_t *channel;
1270 DMA_DeviceAttribute_t *devAttr;
1271 int numDst1Descriptors;
1272 int numDst2Descriptors;
1274 size_t ringBytesRequired;
1277 channel = HandleToChannel(handle);
1278 if (channel == NULL) {
1282 devAttr = &DMA_gDeviceAttribute[channel->devType];
1284 /* Figure out how many descriptors we need. */
1286 /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
1287 /* srcData, dstData, numBytes); */
1289 numDst1Descriptors =
1290 dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
1291 (void *)dstData1, numBytes);
1292 if (numDst1Descriptors < 0) {
1295 numDst2Descriptors =
1296 dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
1297 (void *)dstData2, numBytes);
1298 if (numDst2Descriptors < 0) {
1301 numDescriptors = numDst1Descriptors + numDst2Descriptors;
1302 /* printk("numDescriptors: %d\n", numDescriptors); */
1304 /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
1307 ringBytesRequired = dmacHw_descriptorLen(numDescriptors);
1309 /* printk("ringBytesRequired: %d\n", ringBytesRequired); */
1311 if (ringBytesRequired > devAttr->ring.bytesAllocated) {
1312 /* Make sure that this code path is never taken from interrupt context. */
1313 /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
1314 /* allocation needs to have already been done. */
1318 /* Free the old descriptor ring and allocate a new one. */
1320 dma_free_descriptor_ring(&devAttr->ring);
1322 /* And allocate a new one. */
1325 dma_alloc_descriptor_ring(&devAttr->ring,
1329 "%s: dma_alloc_descriptor_ring(%d) failed\n",
1330 __func__, ringBytesRequired);
1335 /* Setup the descriptor for this transfer. Since this function is used with */
1336 /* CONTINUOUS DMA operations, we need to reinitialize every time, otherwise */
1337 /* setDataDescriptor will keep trying to append onto the end. */
1339 if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
1340 devAttr->ring.physAddr,
1341 devAttr->ring.bytesAllocated,
1342 numDescriptors) < 0) {
1343 printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n", __func__);
1347 /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
1348 /* as last time, then we don't need to call setDataDescriptor again. */
1350 if (dmacHw_setDataDescriptor(&devAttr->config,
1351 devAttr->ring.virtAddr,
1353 (void *)dstData1, numBytes) < 0) {
1354 printk(KERN_ERR "%s: dmacHw_setDataDescriptor 1 failed\n",
1358 if (dmacHw_setDataDescriptor(&devAttr->config,
1359 devAttr->ring.virtAddr,
1361 (void *)dstData2, numBytes) < 0) {
1362 printk(KERN_ERR "%s: dmacHw_setDataDescriptor 2 failed\n",
1367 /* You should use dma_start_transfer rather than dma_transfer_xxx so we don't */
1368 /* try to make the 'prev' variables right. */
1370 devAttr->prevSrcData = 0;
1371 devAttr->prevDstData = 0;
1372 devAttr->prevNumBytes = 0;
1374 return numDescriptors;
1377 EXPORT_SYMBOL(dma_alloc_double_dst_descriptors);
1379 /****************************************************************************/
1381 * Initiates a transfer when the descriptors have already been setup.
1383 * This is a special case, and normally, the dma_transfer_xxx functions should
1387 * 0 Transfer was started successfully
1388 * -ENODEV Invalid handle
1390 /****************************************************************************/
1392 int dma_start_transfer(DMA_Handle_t handle)
1394 DMA_Channel_t *channel;
1395 DMA_DeviceAttribute_t *devAttr;
1397 channel = HandleToChannel(handle);
1398 if (channel == NULL) {
1401 devAttr = &DMA_gDeviceAttribute[channel->devType];
1403 dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
1404 devAttr->ring.virtAddr);
1406 /* Since we got this far, everything went successfully */
1411 EXPORT_SYMBOL(dma_start_transfer);
1413 /****************************************************************************/
1415 * Stops a previously started DMA transfer.
1418 * 0 Transfer was stopped successfully
1419 * -ENODEV Invalid handle
1421 /****************************************************************************/
1423 int dma_stop_transfer(DMA_Handle_t handle)
1425 DMA_Channel_t *channel;
1427 channel = HandleToChannel(handle);
1428 if (channel == NULL) {
1432 dmacHw_stopTransfer(channel->dmacHwHandle);
1437 EXPORT_SYMBOL(dma_stop_transfer);
1439 /****************************************************************************/
1441 * Waits for a DMA to complete by polling. This function is only intended
1442 * to be used for testing. Interrupts should be used for most DMA operations.
1444 /****************************************************************************/
1446 int dma_wait_transfer_done(DMA_Handle_t handle)
1448 DMA_Channel_t *channel;
1449 dmacHw_TRANSFER_STATUS_e status;
1451 channel = HandleToChannel(handle);
1452 if (channel == NULL) {
1457 dmacHw_transferCompleted(channel->dmacHwHandle)) ==
1458 dmacHw_TRANSFER_STATUS_BUSY) {
1462 if (status == dmacHw_TRANSFER_STATUS_ERROR) {
1463 printk(KERN_ERR "%s: DMA transfer failed\n", __func__);
1469 EXPORT_SYMBOL(dma_wait_transfer_done);
1471 /****************************************************************************/
1473 * Initiates a DMA, allocating the descriptors as required.
1476 * 0 Transfer was started successfully
1477 * -EINVAL Invalid device type for this kind of transfer
1478 * (i.e. the device is _DEV_TO_MEM and not _MEM_TO_DEV)
1480 /****************************************************************************/
1482 int dma_transfer(DMA_Handle_t handle, /* DMA Handle */
1483 dmacHw_TRANSFER_TYPE_e transferType, /* Type of transfer being performed */
1484 dma_addr_t srcData, /* Place to get data to write to device */
1485 dma_addr_t dstData, /* Pointer to device data address */
1486 size_t numBytes /* Number of bytes to transfer to the device */
1488 DMA_Channel_t *channel;
1489 DMA_DeviceAttribute_t *devAttr;
1492 channel = HandleToChannel(handle);
1493 if (channel == NULL) {
1497 devAttr = &DMA_gDeviceAttribute[channel->devType];
1499 if (devAttr->config.transferType != transferType) {
1503 /* We keep track of the information about the previous request for this */
1504 /* device, and if the attributes match, then we can use the descriptors we setup */
1505 /* the last time, and not have to reinitialize everything. */
1509 dma_alloc_descriptors(handle, transferType, srcData,
1516 /* And kick off the transfer */
1518 devAttr->numBytes = numBytes;
1519 devAttr->transferStartTime = timer_get_tick_count();
1521 dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
1522 devAttr->ring.virtAddr);
1524 /* Since we got this far, everything went successfully */
1529 EXPORT_SYMBOL(dma_transfer);
1531 /****************************************************************************/
1533 * Set the callback function which will be called when a transfer completes.
1534 * If a NULL callback function is set, then no callback will occur.
1536 * @note @a devHandler will be called from IRQ context.
1540 * -ENODEV - Device handed in is invalid.
1542 /****************************************************************************/
1544 int dma_set_device_handler(DMA_Device_t dev, /* Device to set the callback for. */
1545 DMA_DeviceHandler_t devHandler, /* Function to call when the DMA completes */
1546 void *userData /* Pointer which will be passed to devHandler. */
1548 DMA_DeviceAttribute_t *devAttr;
1549 unsigned long flags;
1551 if (!IsDeviceValid(dev)) {
1554 devAttr = &DMA_gDeviceAttribute[dev];
1556 local_irq_save(flags);
1558 devAttr->userData = userData;
1559 devAttr->devHandler = devHandler;
1561 local_irq_restore(flags);
1566 EXPORT_SYMBOL(dma_set_device_handler);
1568 /****************************************************************************/
1570 * Initializes a memory mapping structure
1572 /****************************************************************************/
1574 int dma_init_mem_map(DMA_MemMap_t *memMap)
1576 memset(memMap, 0, sizeof(*memMap));
1578 sema_init(&memMap->lock, 1);
1583 EXPORT_SYMBOL(dma_init_mem_map);
1585 /****************************************************************************/
1587 * Releases any memory currently being held by a memory mapping structure.
1589 /****************************************************************************/
1591 int dma_term_mem_map(DMA_MemMap_t *memMap)
1593 down(&memMap->lock); /* Just being paranoid */
1595 /* Free up any allocated memory */
1598 memset(memMap, 0, sizeof(*memMap));
1603 EXPORT_SYMBOL(dma_term_mem_map);
1605 /****************************************************************************/
1607 * Looks at a memory address and categorizes it.
1609 * @return One of the values from the DMA_MemType_t enumeration.
1611 /****************************************************************************/
1613 DMA_MemType_t dma_mem_type(void *addr)
1615 unsigned long addrVal = (unsigned long)addr;
1617 if (addrVal >= VMALLOC_END) {
1618 /* NOTE: DMA virtual memory space starts at 0xFFxxxxxx */
1620 /* dma_alloc_xxx pages are physically and virtually contiguous */
1622 return DMA_MEM_TYPE_DMA;
1625 /* Technically, we could add one more classification. Addresses between VMALLOC_END */
1626 /* and the beginning of the DMA virtual address could be considered to be I/O space. */
1627 /* Right now, nobody cares about this particular classification, so we ignore it. */
1629 if (is_vmalloc_addr(addr)) {
1630 /* Address comes from the vmalloc'd region. Pages are virtually */
1631 /* contiguous but NOT physically contiguous */
1633 return DMA_MEM_TYPE_VMALLOC;
1636 if (addrVal >= PAGE_OFFSET) {
1637 /* PAGE_OFFSET is typically 0xC0000000 */
1639 /* kmalloc'd pages are physically contiguous */
1641 return DMA_MEM_TYPE_KMALLOC;
1644 return DMA_MEM_TYPE_USER;
1647 EXPORT_SYMBOL(dma_mem_type);
1649 /****************************************************************************/
1651 * Looks at a memory address and determines if we support DMA'ing to/from
1652 * that type of memory.
1655 * return value != 0 means dma supported
1656 * return value == 0 means dma not supported
1658 /****************************************************************************/
1660 int dma_mem_supports_dma(void *addr)
1662 DMA_MemType_t memType = dma_mem_type(addr);
1664 return (memType == DMA_MEM_TYPE_DMA)
1665 #if ALLOW_MAP_OF_KMALLOC_MEMORY
1666 || (memType == DMA_MEM_TYPE_KMALLOC)
1668 || (memType == DMA_MEM_TYPE_USER);
1671 EXPORT_SYMBOL(dma_mem_supports_dma);
1673 /****************************************************************************/
1675 * Maps in a memory region such that it can be used for performing a DMA.
1679 /****************************************************************************/
1681 int dma_map_start(DMA_MemMap_t *memMap, /* Stores state information about the map */
1682 enum dma_data_direction dir /* Direction that the mapping will be going */
1686 down(&memMap->lock);
1688 DMA_MAP_PRINT("memMap: %p\n", memMap);
1690 if (memMap->inUse) {
1691 printk(KERN_ERR "%s: memory map %p is already being used\n",
1699 memMap->numRegionsUsed = 0;
1705 DMA_MAP_PRINT("returning %d", rc);
1712 EXPORT_SYMBOL(dma_map_start);
1714 /****************************************************************************/
1716 * Adds a segment of memory to a memory map. Each segment is both
1717 * physically and virtually contiguous.
1719 * @return 0 on success, error code otherwise.
1721 /****************************************************************************/
1723 static int dma_map_add_segment(DMA_MemMap_t *memMap, /* Stores state information about the map */
1724 DMA_Region_t *region, /* Region that the segment belongs to */
1725 void *virtAddr, /* Virtual address of the segment being added */
1726 dma_addr_t physAddr, /* Physical address of the segment being added */
1727 size_t numBytes /* Number of bytes of the segment being added */
1729 DMA_Segment_t *segment;
1731 DMA_MAP_PRINT("memMap:%p va:%p pa:0x%x #:%d\n", memMap, virtAddr,
1732 physAddr, numBytes);
1736 if (((unsigned long)virtAddr < (unsigned long)region->virtAddr)
1737 || (((unsigned long)virtAddr + numBytes)) >
1738 ((unsigned long)region->virtAddr + region->numBytes)) {
1740 "%s: virtAddr %p is outside region @ %p len: %d\n",
1741 __func__, virtAddr, region->virtAddr, region->numBytes);
1745 if (region->numSegmentsUsed > 0) {
1746 /* Check to see if this segment is physically contiguous with the previous one */
1748 segment = ®ion->segment[region->numSegmentsUsed - 1];
1750 if ((segment->physAddr + segment->numBytes) == physAddr) {
1751 /* It is - just add on to the end */
1753 DMA_MAP_PRINT("appending %d bytes to last segment\n",
1756 segment->numBytes += numBytes;
1762 /* Reallocate to hold more segments, if required. */
1764 if (region->numSegmentsUsed >= region->numSegmentsAllocated) {
1765 DMA_Segment_t *newSegment;
1767 region->numSegmentsAllocated * sizeof(*newSegment);
1768 int newAlloc = region->numSegmentsAllocated + 4;
1769 size_t newSize = newAlloc * sizeof(*newSegment);
1771 newSegment = kmalloc(newSize, GFP_KERNEL);
1772 if (newSegment == NULL) {
1775 memcpy(newSegment, region->segment, oldSize);
1776 memset(&((uint8_t *) newSegment)[oldSize], 0,
1778 kfree(region->segment);
1780 region->numSegmentsAllocated = newAlloc;
1781 region->segment = newSegment;
1784 segment = ®ion->segment[region->numSegmentsUsed];
1785 region->numSegmentsUsed++;
1787 segment->virtAddr = virtAddr;
1788 segment->physAddr = physAddr;
1789 segment->numBytes = numBytes;
1791 DMA_MAP_PRINT("returning success\n");
1796 /****************************************************************************/
1798 * Adds a region of memory to a memory map. Each region is virtually
1799 * contiguous, but not necessarily physically contiguous.
1801 * @return 0 on success, error code otherwise.
1803 /****************************************************************************/
1805 int dma_map_add_region(DMA_MemMap_t *memMap, /* Stores state information about the map */
1806 void *mem, /* Virtual address that we want to get a map of */
1807 size_t numBytes /* Number of bytes being mapped */
1809 unsigned long addr = (unsigned long)mem;
1810 unsigned int offset;
1812 DMA_Region_t *region;
1813 dma_addr_t physAddr;
1815 down(&memMap->lock);
1817 DMA_MAP_PRINT("memMap:%p va:%p #:%d\n", memMap, mem, numBytes);
1819 if (!memMap->inUse) {
1820 printk(KERN_ERR "%s: Make sure you call dma_map_start first\n",
1826 /* Reallocate to hold more regions. */
1828 if (memMap->numRegionsUsed >= memMap->numRegionsAllocated) {
1829 DMA_Region_t *newRegion;
1831 memMap->numRegionsAllocated * sizeof(*newRegion);
1832 int newAlloc = memMap->numRegionsAllocated + 4;
1833 size_t newSize = newAlloc * sizeof(*newRegion);
1835 newRegion = kmalloc(newSize, GFP_KERNEL);
1836 if (newRegion == NULL) {
1840 memcpy(newRegion, memMap->region, oldSize);
1841 memset(&((uint8_t *) newRegion)[oldSize], 0, newSize - oldSize);
1843 kfree(memMap->region);
1845 memMap->numRegionsAllocated = newAlloc;
1846 memMap->region = newRegion;
1849 region = &memMap->region[memMap->numRegionsUsed];
1850 memMap->numRegionsUsed++;
1852 offset = addr & ~PAGE_MASK;
1854 region->memType = dma_mem_type(mem);
1855 region->virtAddr = mem;
1856 region->numBytes = numBytes;
1857 region->numSegmentsUsed = 0;
1858 region->numLockedPages = 0;
1859 region->lockedPages = NULL;
1861 switch (region->memType) {
1862 case DMA_MEM_TYPE_VMALLOC:
1864 atomic_inc(&gDmaStatMemTypeVmalloc);
1866 /* printk(KERN_ERR "%s: vmalloc'd pages are not supported\n", __func__); */
1868 /* vmalloc'd pages are not physically contiguous */
1874 case DMA_MEM_TYPE_KMALLOC:
1876 atomic_inc(&gDmaStatMemTypeKmalloc);
1878 /* kmalloc'd pages are physically contiguous, so they'll have exactly */
1881 #if ALLOW_MAP_OF_KMALLOC_MEMORY
1883 dma_map_single(NULL, mem, numBytes, memMap->dir);
1884 rc = dma_map_add_segment(memMap, region, mem, physAddr,
1892 case DMA_MEM_TYPE_DMA:
1894 /* dma_alloc_xxx pages are physically contiguous */
1896 atomic_inc(&gDmaStatMemTypeCoherent);
1898 physAddr = (vmalloc_to_pfn(mem) << PAGE_SHIFT) + offset;
1900 dma_sync_single_for_cpu(NULL, physAddr, numBytes,
1902 rc = dma_map_add_segment(memMap, region, mem, physAddr,
1907 case DMA_MEM_TYPE_USER:
1909 size_t firstPageOffset;
1910 size_t firstPageSize;
1911 struct page **pages;
1912 struct task_struct *userTask;
1914 atomic_inc(&gDmaStatMemTypeUser);
1917 /* If the pages are user pages, then the dma_mem_map_set_user_task function */
1918 /* must have been previously called. */
1920 if (memMap->userTask == NULL) {
1922 "%s: must call dma_mem_map_set_user_task when using user-mode memory\n",
1927 /* User pages need to be locked. */
1930 (unsigned long)region->virtAddr & (PAGE_SIZE - 1);
1931 firstPageSize = PAGE_SIZE - firstPageOffset;
1933 region->numLockedPages = (firstPageOffset
1934 + region->numBytes +
1935 PAGE_SIZE - 1) / PAGE_SIZE;
1937 kmalloc(region->numLockedPages *
1938 sizeof(struct page *), GFP_KERNEL);
1940 if (pages == NULL) {
1941 region->numLockedPages = 0;
1945 userTask = memMap->userTask;
1947 down_read(&userTask->mm->mmap_sem);
1948 rc = get_user_pages(userTask, /* task */
1949 userTask->mm, /* mm */
1950 (unsigned long)region->virtAddr, /* start */
1951 region->numLockedPages, /* len */
1952 memMap->dir == DMA_FROM_DEVICE, /* write */
1954 pages, /* pages (array of pointers to page) */
1956 up_read(&userTask->mm->mmap_sem);
1958 if (rc != region->numLockedPages) {
1960 region->numLockedPages = 0;
1966 uint8_t *virtAddr = region->virtAddr;
1967 size_t bytesRemaining;
1970 rc = 0; /* Since get_user_pages returns +ve number */
1972 region->lockedPages = pages;
1974 /* We've locked the user pages. Now we need to walk them and figure */
1975 /* out the physical addresses. */
1977 /* The first page may be partial */
1979 dma_map_add_segment(memMap,
1982 PFN_PHYS(page_to_pfn
1987 virtAddr += firstPageSize;
1989 region->numBytes - firstPageSize;
1992 pageIdx < region->numLockedPages;
1994 size_t bytesThisPage =
1996 PAGE_SIZE ? PAGE_SIZE :
2000 ("pageIdx:%d pages[pageIdx]=%p pfn=%u phys=%u\n",
2001 pageIdx, pages[pageIdx],
2002 page_to_pfn(pages[pageIdx]),
2003 PFN_PHYS(page_to_pfn
2006 dma_map_add_segment(memMap,
2009 PFN_PHYS(page_to_pfn
2014 virtAddr += bytesThisPage;
2015 bytesRemaining -= bytesThisPage;
2020 "%s: User mode pages are not yet supported\n",
2023 /* user pages are not physically contiguous */
2032 printk(KERN_ERR "%s: Unsupported memory type: %d\n",
2033 __func__, region->memType);
2041 memMap->numRegionsUsed--;
2046 DMA_MAP_PRINT("returning %d\n", rc);
2053 EXPORT_SYMBOL(dma_map_add_segment);
2055 /****************************************************************************/
2057 * Maps in a memory region such that it can be used for performing a DMA.
2059 * @return 0 on success, error code otherwise.
2061 /****************************************************************************/
2063 int dma_map_mem(DMA_MemMap_t *memMap, /* Stores state information about the map */
2064 void *mem, /* Virtual address that we want to get a map of */
2065 size_t numBytes, /* Number of bytes being mapped */
2066 enum dma_data_direction dir /* Direction that the mapping will be going */
2070 rc = dma_map_start(memMap, dir);
2072 rc = dma_map_add_region(memMap, mem, numBytes);
2074 /* Since the add fails, this function will fail, and the caller won't */
2075 /* call unmap, so we need to do it here. */
2077 dma_unmap(memMap, 0);
2084 EXPORT_SYMBOL(dma_map_mem);
2086 /****************************************************************************/
2088 * Setup a descriptor ring for a given memory map.
2090 * It is assumed that the descriptor ring has already been initialized, and
2091 * this routine will only reallocate a new descriptor ring if the existing
2094 * @return 0 on success, error code otherwise.
2096 /****************************************************************************/
2098 int dma_map_create_descriptor_ring(DMA_Device_t dev, /* DMA device (where the ring is stored) */
2099 DMA_MemMap_t *memMap, /* Memory map that will be used */
2100 dma_addr_t devPhysAddr /* Physical address of device */
2104 DMA_DeviceAttribute_t *devAttr;
2105 DMA_Region_t *region;
2106 DMA_Segment_t *segment;
2107 dma_addr_t srcPhysAddr;
2108 dma_addr_t dstPhysAddr;
2112 devAttr = &DMA_gDeviceAttribute[dev];
2114 down(&memMap->lock);
2116 /* Figure out how many descriptors we need */
2119 for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2120 region = &memMap->region[regionIdx];
2122 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2124 segment = ®ion->segment[segmentIdx];
2126 if (memMap->dir == DMA_TO_DEVICE) {
2127 srcPhysAddr = segment->physAddr;
2128 dstPhysAddr = devPhysAddr;
2130 srcPhysAddr = devPhysAddr;
2131 dstPhysAddr = segment->physAddr;
2135 dma_calculate_descriptor_count(dev, srcPhysAddr,
2141 "%s: dma_calculate_descriptor_count failed: %d\n",
2145 numDescriptors += rc;
2149 /* Adjust the size of the ring, if it isn't big enough */
2151 if (numDescriptors > devAttr->ring.descriptorsAllocated) {
2152 dma_free_descriptor_ring(&devAttr->ring);
2154 dma_alloc_descriptor_ring(&devAttr->ring,
2158 "%s: dma_alloc_descriptor_ring failed: %d\n",
2164 dma_init_descriptor_ring(&devAttr->ring,
2168 "%s: dma_init_descriptor_ring failed: %d\n",
2174 /* Populate the descriptors */
2176 for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2177 region = &memMap->region[regionIdx];
2179 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2181 segment = ®ion->segment[segmentIdx];
2183 if (memMap->dir == DMA_TO_DEVICE) {
2184 srcPhysAddr = segment->physAddr;
2185 dstPhysAddr = devPhysAddr;
2187 srcPhysAddr = devPhysAddr;
2188 dstPhysAddr = segment->physAddr;
2192 dma_add_descriptors(&devAttr->ring, dev,
2193 srcPhysAddr, dstPhysAddr,
2197 "%s: dma_add_descriptors failed: %d\n",
2212 EXPORT_SYMBOL(dma_map_create_descriptor_ring);
2214 /****************************************************************************/
2216 * Maps in a memory region such that it can be used for performing a DMA.
2220 /****************************************************************************/
2222 int dma_unmap(DMA_MemMap_t *memMap, /* Stores state information about the map */
2223 int dirtied /* non-zero if any of the pages were modified */
2229 DMA_Region_t *region;
2230 DMA_Segment_t *segment;
2232 down(&memMap->lock);
2234 for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
2235 region = &memMap->region[regionIdx];
2237 for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
2239 segment = ®ion->segment[segmentIdx];
2241 switch (region->memType) {
2242 case DMA_MEM_TYPE_VMALLOC:
2245 "%s: vmalloc'd pages are not yet supported\n",
2251 case DMA_MEM_TYPE_KMALLOC:
2253 #if ALLOW_MAP_OF_KMALLOC_MEMORY
2254 dma_unmap_single(NULL,
2262 case DMA_MEM_TYPE_DMA:
2264 dma_sync_single_for_cpu(NULL,
2273 case DMA_MEM_TYPE_USER:
2275 /* Nothing to do here. */
2283 "%s: Unsupported memory type: %d\n",
2284 __func__, region->memType);
2290 segment->virtAddr = NULL;
2291 segment->physAddr = 0;
2292 segment->numBytes = 0;
2295 if (region->numLockedPages > 0) {
2298 /* Some user pages were locked. We need to go and unlock them now. */
2300 for (pageIdx = 0; pageIdx < region->numLockedPages;
2303 region->lockedPages[pageIdx];
2305 if (memMap->dir == DMA_FROM_DEVICE) {
2308 page_cache_release(page);
2310 kfree(region->lockedPages);
2311 region->numLockedPages = 0;
2312 region->lockedPages = NULL;
2315 region->memType = DMA_MEM_TYPE_NONE;
2316 region->virtAddr = NULL;
2317 region->numBytes = 0;
2318 region->numSegmentsUsed = 0;
2320 memMap->userTask = NULL;
2321 memMap->numRegionsUsed = 0;
2330 EXPORT_SYMBOL(dma_unmap);