drivers/soc/qcom/smd.c

   1 /*
   2  * Copyright (c) 2015, Sony Mobile Communications AB.
   3  * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 and
   7  * only version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  */
  14
  15 #include <linux/interrupt.h>
  16 #include <linux/io.h>
  17 #include <linux/mfd/syscon.h>
  18 #include <linux/module.h>
  19 #include <linux/of_irq.h>
  20 #include <linux/of_platform.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/regmap.h>
  23 #include <linux/sched.h>
  24 #include <linux/slab.h>
  25 #include <linux/soc/qcom/smd.h>
  26 #include <linux/soc/qcom/smem.h>
  27 #include <linux/wait.h>
  28
  29 /*
  30  * The Qualcomm Shared Memory communication solution provides point-to-point
  31  * channels for clients to send and receive streaming or packet based data.
  32  *
  33  * Each channel consists of a control item (channel info) and a ring buffer
  34  * pair. The channel info carry information related to channel state, flow
  35  * control and the offsets within the ring buffer.
  36  *
  37  * All allocated channels are listed in an allocation table, identifying the
  38  * pair of items by name, type and remote processor.
  39  *
  40  * Upon creating a new channel the remote processor allocates channel info and
  41  * ring buffer items from the smem heap and populate the allocation table. An
  42  * interrupt is sent to the other end of the channel and a scan for new
  43  * channels should be done. A channel never goes away, it will only change
  44  * state.
  45  *
  46  * The remote processor signals it intent for bring up the communication
  47  * channel by setting the state of its end of the channel to "opening" and
  48  * sends out an interrupt. We detect this change and register a smd device to
  49  * consume the channel. Upon finding a consumer we finish the handshake and the
  50  * channel is up.
  51  *
  52  * Upon closing a channel, the remote processor will update the state of its
  53  * end of the channel and signal us, we will then unregister any attached
  54  * device and close our end of the channel.
  55  *
  56  * Devices attached to a channel can use the qcom_smd_send function to push
  57  * data to the channel, this is done by copying the data into the tx ring
  58  * buffer, updating the pointers in the channel info and signaling the remote
  59  * processor.
  60  *
  61  * The remote processor does the equivalent when it transfer data and upon
  62  * receiving the interrupt we check the channel info for new data and delivers
  63  * this to the attached device. If the device is not ready to receive the data
  64  * we leave it in the ring buffer for now.
  65  */
  66
  67 struct smd_channel_info;
  68 struct smd_channel_info_word;
  69
  70 #define SMD_ALLOC_TBL_COUNT     2
  71 #define SMD_ALLOC_TBL_SIZE      64
  72
  73 /*
  74  * This lists the various smem heap items relevant for the allocation table and
  75  * smd channel entries.
  76  */
  77 static const struct {
  78         unsigned alloc_tbl_id;
  79         unsigned info_base_id;
  80         unsigned fifo_base_id;
  81 } smem_items[SMD_ALLOC_TBL_COUNT] = {
  82         {
  83                 .alloc_tbl_id = 13,
  84                 .info_base_id = 14,
  85                 .fifo_base_id = 338
  86         },
  87         {
  88                 .alloc_tbl_id = 14,
  89                 .info_base_id = 266,
  90                 .fifo_base_id = 202,
  91         },
  92 };
  93
  94 /**
  95  * struct qcom_smd_edge - representing a remote processor
  96  * @smd:                handle to qcom_smd
  97  * @of_node:            of_node handle for information related to this edge
  98  * @edge_id:            identifier of this edge
  99  * @remote_pid:         identifier of remote processor
 100  * @irq:                interrupt for signals on this edge
 101  * @ipc_regmap:         regmap handle holding the outgoing ipc register
 102  * @ipc_offset:         offset within @ipc_regmap of the register for ipc
 103  * @ipc_bit:            bit in the register at @ipc_offset of @ipc_regmap
 104  * @channels:           list of all channels detected on this edge
 105  * @channels_lock:      guard for modifications of @channels
 106  * @allocated:          array of bitmaps representing already allocated channels
 107  * @need_rescan:        flag that the @work needs to scan smem for new channels
 108  * @smem_available:     last available amount of smem triggering a channel scan
 109  * @work:               work item for edge house keeping
 110  */
 111 struct qcom_smd_edge {
 112         struct qcom_smd *smd;
 113         struct device_node *of_node;
 114         unsigned edge_id;
 115         unsigned remote_pid;
 116
 117         int irq;
 118
 119         struct regmap *ipc_regmap;
 120         int ipc_offset;
 121         int ipc_bit;
 122
 123         struct list_head channels;
 124         spinlock_t channels_lock;
 125
 126         DECLARE_BITMAP(allocated[SMD_ALLOC_TBL_COUNT], SMD_ALLOC_TBL_SIZE);
 127
 128         bool need_rescan;
 129         unsigned smem_available;
 130
 131         struct work_struct work;
 132 };
 133
 134 /*
 135  * SMD channel states.
 136  */
 137 enum smd_channel_state {
 138         SMD_CHANNEL_CLOSED,
 139         SMD_CHANNEL_OPENING,
 140         SMD_CHANNEL_OPENED,
 141         SMD_CHANNEL_FLUSHING,
 142         SMD_CHANNEL_CLOSING,
 143         SMD_CHANNEL_RESET,
 144         SMD_CHANNEL_RESET_OPENING
 145 };
 146
 147 /**
 148  * struct qcom_smd_channel - smd channel struct
 149  * @edge:               qcom_smd_edge this channel is living on
 150  * @qsdev:              reference to a associated smd client device
 151  * @name:               name of the channel
 152  * @state:              local state of the channel
 153  * @remote_state:       remote state of the channel
 154  * @tx_info:            byte aligned outgoing channel info
 155  * @rx_info:            byte aligned incoming channel info
 156  * @tx_info_word:       word aligned outgoing channel info
 157  * @rx_info_word:       word aligned incoming channel info
 158  * @tx_lock:            lock to make writes to the channel mutually exclusive
 159  * @fblockread_event:   wakeup event tied to tx fBLOCKREADINTR
 160  * @tx_fifo:            pointer to the outgoing ring buffer
 161  * @rx_fifo:            pointer to the incoming ring buffer
 162  * @fifo_size:          size of each ring buffer
 163  * @bounce_buffer:      bounce buffer for reading wrapped packets
 164  * @cb:                 callback function registered for this channel
 165  * @recv_lock:          guard for rx info modifications and cb pointer
 166  * @pkt_size:           size of the currently handled packet
 167  * @list:               lite entry for @channels in qcom_smd_edge
 168  */
 169 struct qcom_smd_channel {
 170         struct qcom_smd_edge *edge;
 171
 172         struct qcom_smd_device *qsdev;
 173
 174         char *name;
 175         enum smd_channel_state state;
 176         enum smd_channel_state remote_state;
 177
 178         struct smd_channel_info *tx_info;
 179         struct smd_channel_info *rx_info;
 180
 181         struct smd_channel_info_word *tx_info_word;
 182         struct smd_channel_info_word *rx_info_word;
 183
 184         struct mutex tx_lock;
 185         wait_queue_head_t fblockread_event;
 186
 187         void *tx_fifo;
 188         void *rx_fifo;
 189         int fifo_size;
 190
 191         void *bounce_buffer;
 192         int (*cb)(struct qcom_smd_device *, const void *, size_t);
 193
 194         spinlock_t recv_lock;
 195
 196         int pkt_size;
 197
 198         struct list_head list;
 199 };
 200
 201 /**
 202  * struct qcom_smd - smd struct
 203  * @dev:        device struct
 204  * @num_edges:  number of entries in @edges
 205  * @edges:      array of edges to be handled
 206  */
 207 struct qcom_smd {
 208         struct device *dev;
 209
 210         unsigned num_edges;
 211         struct qcom_smd_edge edges[0];
 212 };
 213
 214 /*
 215  * Format of the smd_info smem items, for byte aligned channels.
 216  */
 217 struct smd_channel_info {
 218         u32 state;
 219         u8  fDSR;
 220         u8  fCTS;
 221         u8  fCD;
 222         u8  fRI;
 223         u8  fHEAD;
 224         u8  fTAIL;
 225         u8  fSTATE;
 226         u8  fBLOCKREADINTR;
 227         u32 tail;
 228         u32 head;
 229 };
 230
 231 /*
 232  * Format of the smd_info smem items, for word aligned channels.
 233  */
 234 struct smd_channel_info_word {
 235         u32 state;
 236         u32 fDSR;
 237         u32 fCTS;
 238         u32 fCD;
 239         u32 fRI;
 240         u32 fHEAD;
 241         u32 fTAIL;
 242         u32 fSTATE;
 243         u32 fBLOCKREADINTR;
 244         u32 tail;
 245         u32 head;
 246 };
 247
 248 #define GET_RX_CHANNEL_INFO(channel, param) \
 249         (channel->rx_info_word ? \
 250                 channel->rx_info_word->param : \
 251                 channel->rx_info->param)
 252
 253 #define SET_RX_CHANNEL_INFO(channel, param, value) \
 254         (channel->rx_info_word ? \
 255                 (channel->rx_info_word->param = value) : \
 256                 (channel->rx_info->param = value))
 257
 258 #define GET_TX_CHANNEL_INFO(channel, param) \
 259         (channel->tx_info_word ? \
 260                 channel->tx_info_word->param : \
 261                 channel->tx_info->param)
 262
 263 #define SET_TX_CHANNEL_INFO(channel, param, value) \
 264         (channel->tx_info_word ? \
 265                 (channel->tx_info_word->param = value) : \
 266                 (channel->tx_info->param = value))
 267
 268 /**
 269  * struct qcom_smd_alloc_entry - channel allocation entry
 270  * @name:       channel name
 271  * @cid:        channel index
 272  * @flags:      channel flags and edge id
 273  * @ref_count:  reference count of the channel
 274  */
 275 struct qcom_smd_alloc_entry {
 276         u8 name[20];
 277         u32 cid;
 278         u32 flags;
 279         u32 ref_count;
 280 } __packed;
 281
 282 #define SMD_CHANNEL_FLAGS_EDGE_MASK     0xff
 283 #define SMD_CHANNEL_FLAGS_STREAM        BIT(8)
 284 #define SMD_CHANNEL_FLAGS_PACKET        BIT(9)
 285
 286 /*
 287  * Each smd packet contains a 20 byte header, with the first 4 being the length
 288  * of the packet.
 289  */
 290 #define SMD_PACKET_HEADER_LEN   20
 291
 292 /*
 293  * Signal the remote processor associated with 'channel'.
 294  */
 295 static void qcom_smd_signal_channel(struct qcom_smd_channel *channel)
 296 {
 297         struct qcom_smd_edge *edge = channel->edge;
 298
 299         regmap_write(edge->ipc_regmap, edge->ipc_offset, BIT(edge->ipc_bit));
 300 }
 301
 302 /*
 303  * Initialize the tx channel info
 304  */
 305 static void qcom_smd_channel_reset(struct qcom_smd_channel *channel)
 306 {
 307         SET_TX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED);
 308         SET_TX_CHANNEL_INFO(channel, fDSR, 0);
 309         SET_TX_CHANNEL_INFO(channel, fCTS, 0);
 310         SET_TX_CHANNEL_INFO(channel, fCD, 0);
 311         SET_TX_CHANNEL_INFO(channel, fRI, 0);
 312         SET_TX_CHANNEL_INFO(channel, fHEAD, 0);
 313         SET_TX_CHANNEL_INFO(channel, fTAIL, 0);
 314         SET_TX_CHANNEL_INFO(channel, fSTATE, 1);
 315         SET_TX_CHANNEL_INFO(channel, fBLOCKREADINTR, 1);
 316         SET_TX_CHANNEL_INFO(channel, head, 0);
 317         SET_TX_CHANNEL_INFO(channel, tail, 0);
 318
 319         qcom_smd_signal_channel(channel);
 320
 321         channel->state = SMD_CHANNEL_CLOSED;
 322         channel->pkt_size = 0;
 323 }
 324
 325 /*
 326  * Calculate the amount of data available in the rx fifo
 327  */
 328 static size_t qcom_smd_channel_get_rx_avail(struct qcom_smd_channel *channel)
 329 {
 330         unsigned head;
 331         unsigned tail;
 332
 333         head = GET_RX_CHANNEL_INFO(channel, head);
 334         tail = GET_RX_CHANNEL_INFO(channel, tail);
 335
 336         return (head - tail) & (channel->fifo_size - 1);
 337 }
 338
 339 /*
 340  * Set tx channel state and inform the remote processor
 341  */
 342 static void qcom_smd_channel_set_state(struct qcom_smd_channel *channel,
 343                                        int state)
 344 {
 345         struct qcom_smd_edge *edge = channel->edge;
 346         bool is_open = state == SMD_CHANNEL_OPENED;
 347
 348         if (channel->state == state)
 349                 return;
 350
 351         dev_dbg(edge->smd->dev, "set_state(%s, %d)\n", channel->name, state);
 352
 353         SET_TX_CHANNEL_INFO(channel, fDSR, is_open);
 354         SET_TX_CHANNEL_INFO(channel, fCTS, is_open);
 355         SET_TX_CHANNEL_INFO(channel, fCD, is_open);
 356
 357         SET_TX_CHANNEL_INFO(channel, state, state);
 358         SET_TX_CHANNEL_INFO(channel, fSTATE, 1);
 359
 360         channel->state = state;
 361         qcom_smd_signal_channel(channel);
 362 }
 363
 364 /*
 365  * Copy count bytes of data using 32bit accesses, if that's required.
 366  */
 367 static void smd_copy_to_fifo(void __iomem *_dst,
 368                              const void *_src,
 369                              size_t count,
 370                              bool word_aligned)
 371 {
 372         u32 *dst = (u32 *)_dst;
 373         u32 *src = (u32 *)_src;
 374
 375         if (word_aligned) {
 376                 count /= sizeof(u32);
 377                 while (count--)
 378                         writel_relaxed(*src++, dst++);
 379         } else {
 380                 memcpy_toio(_dst, _src, count);
 381         }
 382 }
 383
 384 /*
 385  * Copy count bytes of data using 32bit accesses, if that is required.
 386  */
 387 static void smd_copy_from_fifo(void *_dst,
 388                                const void __iomem *_src,
 389                                size_t count,
 390                                bool word_aligned)
 391 {
 392         u32 *dst = (u32 *)_dst;
 393         u32 *src = (u32 *)_src;
 394
 395         if (word_aligned) {
 396                 count /= sizeof(u32);
 397                 while (count--)
 398                         *dst++ = readl_relaxed(src++);
 399         } else {
 400                 memcpy_fromio(_dst, _src, count);
 401         }
 402 }
 403
 404 /*
 405  * Read count bytes of data from the rx fifo into buf, but don't advance the
 406  * tail.
 407  */
 408 static size_t qcom_smd_channel_peek(struct qcom_smd_channel *channel,
 409                                     void *buf, size_t count)
 410 {
 411         bool word_aligned;
 412         unsigned tail;
 413         size_t len;
 414
 415         word_aligned = channel->rx_info_word != NULL;
 416         tail = GET_RX_CHANNEL_INFO(channel, tail);
 417
 418         len = min_t(size_t, count, channel->fifo_size - tail);
 419         if (len) {
 420                 smd_copy_from_fifo(buf,
 421                                    channel->rx_fifo + tail,
 422                                    len,
 423                                    word_aligned);
 424         }
 425
 426         if (len != count) {
 427                 smd_copy_from_fifo(buf + len,
 428                                    channel->rx_fifo,
 429                                    count - len,
 430                                    word_aligned);
 431         }
 432
 433         return count;
 434 }
 435
 436 /*
 437  * Advance the rx tail by count bytes.
 438  */
 439 static void qcom_smd_channel_advance(struct qcom_smd_channel *channel,
 440                                      size_t count)
 441 {
 442         unsigned tail;
 443
 444         tail = GET_RX_CHANNEL_INFO(channel, tail);
 445         tail += count;
 446         tail &= (channel->fifo_size - 1);
 447         SET_RX_CHANNEL_INFO(channel, tail, tail);
 448 }
 449
 450 /*
 451  * Read out a single packet from the rx fifo and deliver it to the device
 452  */
 453 static int qcom_smd_channel_recv_single(struct qcom_smd_channel *channel)
 454 {
 455         struct qcom_smd_device *qsdev = channel->qsdev;
 456         unsigned tail;
 457         size_t len;
 458         void *ptr;
 459         int ret;
 460
 461         if (!channel->cb)
 462                 return 0;
 463
 464         tail = GET_RX_CHANNEL_INFO(channel, tail);
 465
 466         /* Use bounce buffer if the data wraps */
 467         if (tail + channel->pkt_size >= channel->fifo_size) {
 468                 ptr = channel->bounce_buffer;
 469                 len = qcom_smd_channel_peek(channel, ptr, channel->pkt_size);
 470         } else {
 471                 ptr = channel->rx_fifo + tail;
 472                 len = channel->pkt_size;
 473         }
 474
 475         ret = channel->cb(qsdev, ptr, len);
 476         if (ret < 0)
 477                 return ret;
 478
 479         /* Only forward the tail if the client consumed the data */
 480         qcom_smd_channel_advance(channel, len);
 481
 482         channel->pkt_size = 0;
 483
 484         return 0;
 485 }
 486
 487 /*
 488  * Per channel interrupt handling
 489  */
 490 static bool qcom_smd_channel_intr(struct qcom_smd_channel *channel)
 491 {
 492         bool need_state_scan = false;
 493         int remote_state;
 494         u32 pktlen;
 495         int avail;
 496         int ret;
 497
 498         /* Handle state changes */
 499         remote_state = GET_RX_CHANNEL_INFO(channel, state);
 500         if (remote_state != channel->remote_state) {
 501                 channel->remote_state = remote_state;
 502                 need_state_scan = true;
 503         }
 504         /* Indicate that we have seen any state change */
 505         SET_RX_CHANNEL_INFO(channel, fSTATE, 0);
 506
 507         /* Signal waiting qcom_smd_send() about the interrupt */
 508         if (!GET_TX_CHANNEL_INFO(channel, fBLOCKREADINTR))
 509                 wake_up_interruptible(&channel->fblockread_event);
 510
 511         /* Don't consume any data until we've opened the channel */
 512         if (channel->state != SMD_CHANNEL_OPENED)
 513                 goto out;
 514
 515         /* Indicate that we've seen the new data */
 516         SET_RX_CHANNEL_INFO(channel, fHEAD, 0);
 517
 518         /* Consume data */
 519         for (;;) {
 520                 avail = qcom_smd_channel_get_rx_avail(channel);
 521
 522                 if (!channel->pkt_size && avail >= SMD_PACKET_HEADER_LEN) {
 523                         qcom_smd_channel_peek(channel, &pktlen, sizeof(pktlen));
 524                         qcom_smd_channel_advance(channel, SMD_PACKET_HEADER_LEN);
 525                         channel->pkt_size = pktlen;
 526                 } else if (channel->pkt_size && avail >= channel->pkt_size) {
 527                         ret = qcom_smd_channel_recv_single(channel);
 528                         if (ret)
 529                                 break;
 530                 } else {
 531                         break;
 532                 }
 533         }
 534
 535         /* Indicate that we have seen and updated tail */
 536         SET_RX_CHANNEL_INFO(channel, fTAIL, 1);
 537
 538         /* Signal the remote that we've consumed the data (if requested) */
 539         if (!GET_RX_CHANNEL_INFO(channel, fBLOCKREADINTR)) {
 540                 /* Ensure ordering of channel info updates */
 541                 wmb();
 542
 543                 qcom_smd_signal_channel(channel);
 544         }
 545
 546 out:
 547         return need_state_scan;
 548 }
 549
 550 /*
 551  * The edge interrupts are triggered by the remote processor on state changes,
 552  * channel info updates or when new channels are created.
 553  */
 554 static irqreturn_t qcom_smd_edge_intr(int irq, void *data)
 555 {
 556         struct qcom_smd_edge *edge = data;
 557         struct qcom_smd_channel *channel;
 558         unsigned available;
 559         bool kick_worker = false;
 560
 561         /*
 562          * Handle state changes or data on each of the channels on this edge
 563          */
 564         spin_lock(&edge->channels_lock);
 565         list_for_each_entry(channel, &edge->channels, list) {
 566                 spin_lock(&channel->recv_lock);
 567                 kick_worker |= qcom_smd_channel_intr(channel);
 568                 spin_unlock(&channel->recv_lock);
 569         }
 570         spin_unlock(&edge->channels_lock);
 571
 572         /*
 573          * Creating a new channel requires allocating an smem entry, so we only
 574          * have to scan if the amount of available space in smem have changed
 575          * since last scan.
 576          */
 577         available = qcom_smem_get_free_space(edge->remote_pid);
 578         if (available != edge->smem_available) {
 579                 edge->smem_available = available;
 580                 edge->need_rescan = true;
 581                 kick_worker = true;
 582         }
 583
 584         if (kick_worker)
 585                 schedule_work(&edge->work);
 586
 587         return IRQ_HANDLED;
 588 }
 589
 590 /*
 591  * Delivers any outstanding packets in the rx fifo, can be used after probe of
 592  * the clients to deliver any packets that wasn't delivered before the client
 593  * was setup.
 594  */
 595 static void qcom_smd_channel_resume(struct qcom_smd_channel *channel)
 596 {
 597         unsigned long flags;
 598
 599         spin_lock_irqsave(&channel->recv_lock, flags);
 600         qcom_smd_channel_intr(channel);
 601         spin_unlock_irqrestore(&channel->recv_lock, flags);
 602 }
 603
 604 /*
 605  * Calculate how much space is available in the tx fifo.
 606  */
 607 static size_t qcom_smd_get_tx_avail(struct qcom_smd_channel *channel)
 608 {
 609         unsigned head;
 610         unsigned tail;
 611         unsigned mask = channel->fifo_size - 1;
 612
 613         head = GET_TX_CHANNEL_INFO(channel, head);
 614         tail = GET_TX_CHANNEL_INFO(channel, tail);
 615
 616         return mask - ((head - tail) & mask);
 617 }
 618
 619 /*
 620  * Write count bytes of data into channel, possibly wrapping in the ring buffer
 621  */
 622 static int qcom_smd_write_fifo(struct qcom_smd_channel *channel,
 623                                const void *data,
 624                                size_t count)
 625 {
 626         bool word_aligned;
 627         unsigned head;
 628         size_t len;
 629
 630         word_aligned = channel->tx_info_word != NULL;
 631         head = GET_TX_CHANNEL_INFO(channel, head);
 632
 633         len = min_t(size_t, count, channel->fifo_size - head);
 634         if (len) {
 635                 smd_copy_to_fifo(channel->tx_fifo + head,
 636                                  data,
 637                                  len,
 638                                  word_aligned);
 639         }
 640
 641         if (len != count) {
 642                 smd_copy_to_fifo(channel->tx_fifo,
 643                                  data + len,
 644                                  count - len,
 645                                  word_aligned);
 646         }
 647
 648         head += count;
 649         head &= (channel->fifo_size - 1);
 650         SET_TX_CHANNEL_INFO(channel, head, head);
 651
 652         return count;
 653 }
 654
 655 /**
 656  * qcom_smd_send - write data to smd channel
 657  * @channel:    channel handle
 658  * @data:       buffer of data to write
 659  * @len:        number of bytes to write
 660  *
 661  * This is a blocking write of len bytes into the channel's tx ring buffer and
 662  * signal the remote end. It will sleep until there is enough space available
 663  * in the tx buffer, utilizing the fBLOCKREADINTR signaling mechanism to avoid
 664  * polling.
 665  */
 666 int qcom_smd_send(struct qcom_smd_channel *channel, const void *data, int len)
 667 {
 668         u32 hdr[5] = {len,};
 669         int tlen = sizeof(hdr) + len;
 670         int ret;
 671
 672         /* Word aligned channels only accept word size aligned data */
 673         if (channel->rx_info_word != NULL && len % 4)
 674                 return -EINVAL;
 675
 676         ret = mutex_lock_interruptible(&channel->tx_lock);
 677         if (ret)
 678                 return ret;
 679
 680         while (qcom_smd_get_tx_avail(channel) < tlen) {
 681                 if (channel->state != SMD_CHANNEL_OPENED) {
 682                         ret = -EPIPE;
 683                         goto out;
 684                 }
 685
 686                 SET_TX_CHANNEL_INFO(channel, fBLOCKREADINTR, 0);
 687
 688                 ret = wait_event_interruptible(channel->fblockread_event,
 689                                        qcom_smd_get_tx_avail(channel) >= tlen ||
 690                                        channel->state != SMD_CHANNEL_OPENED);
 691                 if (ret)
 692                         goto out;
 693
 694                 SET_TX_CHANNEL_INFO(channel, fBLOCKREADINTR, 1);
 695         }
 696
 697         SET_TX_CHANNEL_INFO(channel, fTAIL, 0);
 698
 699         qcom_smd_write_fifo(channel, hdr, sizeof(hdr));
 700         qcom_smd_write_fifo(channel, data, len);
 701
 702         SET_TX_CHANNEL_INFO(channel, fHEAD, 1);
 703
 704         /* Ensure ordering of channel info updates */
 705         wmb();
 706
 707         qcom_smd_signal_channel(channel);
 708
 709 out:
 710         mutex_unlock(&channel->tx_lock);
 711
 712         return ret;
 713 }
 714 EXPORT_SYMBOL(qcom_smd_send);
 715
 716 static struct qcom_smd_device *to_smd_device(struct device *dev)
 717 {
 718         return container_of(dev, struct qcom_smd_device, dev);
 719 }
 720
 721 static struct qcom_smd_driver *to_smd_driver(struct device *dev)
 722 {
 723         struct qcom_smd_device *qsdev = to_smd_device(dev);
 724
 725         return container_of(qsdev->dev.driver, struct qcom_smd_driver, driver);
 726 }
 727
 728 static int qcom_smd_dev_match(struct device *dev, struct device_driver *drv)
 729 {
 730         return of_driver_match_device(dev, drv);
 731 }
 732
 733 /*
 734  * Probe the smd client.
 735  *
 736  * The remote side have indicated that it want the channel to be opened, so
 737  * complete the state handshake and probe our client driver.
 738  */
 739 static int qcom_smd_dev_probe(struct device *dev)
 740 {
 741         struct qcom_smd_device *qsdev = to_smd_device(dev);
 742         struct qcom_smd_driver *qsdrv = to_smd_driver(dev);
 743         struct qcom_smd_channel *channel = qsdev->channel;
 744         size_t bb_size;
 745         int ret;
 746
 747         /*
 748          * Packets are maximum 4k, but reduce if the fifo is smaller
 749          */
 750         bb_size = min(channel->fifo_size, SZ_4K);
 751         channel->bounce_buffer = kmalloc(bb_size, GFP_KERNEL);
 752         if (!channel->bounce_buffer)
 753                 return -ENOMEM;
 754
 755         channel->cb = qsdrv->callback;
 756
 757         qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENING);
 758
 759         qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENED);
 760
 761         ret = qsdrv->probe(qsdev);
 762         if (ret)
 763                 goto err;
 764
 765         qcom_smd_channel_resume(channel);
 766
 767         return 0;
 768
 769 err:
 770         dev_err(&qsdev->dev, "probe failed\n");
 771
 772         channel->cb = NULL;
 773         kfree(channel->bounce_buffer);
 774         channel->bounce_buffer = NULL;
 775
 776         qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED);
 777         return ret;
 778 }
 779
 780 /*
 781  * Remove the smd client.
 782  *
 783  * The channel is going away, for some reason, so remove the smd client and
 784  * reset the channel state.
 785  */
 786 static int qcom_smd_dev_remove(struct device *dev)
 787 {
 788         struct qcom_smd_device *qsdev = to_smd_device(dev);
 789         struct qcom_smd_driver *qsdrv = to_smd_driver(dev);
 790         struct qcom_smd_channel *channel = qsdev->channel;
 791         unsigned long flags;
 792
 793         qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSING);
 794
 795         /*
 796          * Make sure we don't race with the code receiving data.
 797          */
 798         spin_lock_irqsave(&channel->recv_lock, flags);
 799         channel->cb = NULL;
 800         spin_unlock_irqrestore(&channel->recv_lock, flags);
 801
 802         /* Wake up any sleepers in qcom_smd_send() */
 803         wake_up_interruptible(&channel->fblockread_event);
 804
 805         /*
 806          * We expect that the client might block in remove() waiting for any
 807          * outstanding calls to qcom_smd_send() to wake up and finish.
 808          */
 809         if (qsdrv->remove)
 810                 qsdrv->remove(qsdev);
 811
 812         /*
 813          * The client is now gone, cleanup and reset the channel state.
 814          */
 815         channel->qsdev = NULL;
 816         kfree(channel->bounce_buffer);
 817         channel->bounce_buffer = NULL;
 818
 819         qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED);
 820
 821         qcom_smd_channel_reset(channel);
 822
 823         return 0;
 824 }
 825
 826 static struct bus_type qcom_smd_bus = {
 827         .name = "qcom_smd",
 828         .match = qcom_smd_dev_match,
 829         .probe = qcom_smd_dev_probe,
 830         .remove = qcom_smd_dev_remove,
 831 };
 832
 833 /*
 834  * Release function for the qcom_smd_device object.
 835  */
 836 static void qcom_smd_release_device(struct device *dev)
 837 {
 838         struct qcom_smd_device *qsdev = to_smd_device(dev);
 839
 840         kfree(qsdev);
 841 }
 842
 843 /*
 844  * Finds the device_node for the smd child interested in this channel.
 845  */
 846 static struct device_node *qcom_smd_match_channel(struct device_node *edge_node,
 847                                                   const char *channel)
 848 {
 849         struct device_node *child;
 850         const char *name;
 851         const char *key;
 852         int ret;
 853
 854         for_each_available_child_of_node(edge_node, child) {
 855                 key = "qcom,smd-channels";
 856                 ret = of_property_read_string(child, key, &name);
 857                 if (ret) {
 858                         of_node_put(child);
 859                         continue;
 860                 }
 861
 862                 if (strcmp(name, channel) == 0)
 863                         return child;
 864         }
 865
 866         return NULL;
 867 }
 868
 869 /*
 870  * Create a smd client device for channel that is being opened.
 871  */
 872 static int qcom_smd_create_device(struct qcom_smd_channel *channel)
 873 {
 874         struct qcom_smd_device *qsdev;
 875         struct qcom_smd_edge *edge = channel->edge;
 876         struct device_node *node;
 877         struct qcom_smd *smd = edge->smd;
 878         int ret;
 879
 880         if (channel->qsdev)
 881                 return -EEXIST;
 882
 883         node = qcom_smd_match_channel(edge->of_node, channel->name);
 884         if (!node) {
 885                 dev_dbg(smd->dev, "no match for '%s'\n", channel->name);
 886                 return -ENXIO;
 887         }
 888
 889         dev_dbg(smd->dev, "registering '%s'\n", channel->name);
 890
 891         qsdev = kzalloc(sizeof(*qsdev), GFP_KERNEL);
 892         if (!qsdev)
 893                 return -ENOMEM;
 894
 895         dev_set_name(&qsdev->dev, "%s.%s", edge->of_node->name, node->name);
 896         qsdev->dev.parent = smd->dev;
 897         qsdev->dev.bus = &qcom_smd_bus;
 898         qsdev->dev.release = qcom_smd_release_device;
 899         qsdev->dev.of_node = node;
 900
 901         qsdev->channel = channel;
 902
 903         channel->qsdev = qsdev;
 904
 905         ret = device_register(&qsdev->dev);
 906         if (ret) {
 907                 dev_err(smd->dev, "device_register failed: %d\n", ret);
 908                 put_device(&qsdev->dev);
 909         }
 910
 911         return ret;
 912 }
 913
 914 /*
 915  * Destroy a smd client device for a channel that's going away.
 916  */
 917 static void qcom_smd_destroy_device(struct qcom_smd_channel *channel)
 918 {
 919         struct device *dev;
 920
 921         BUG_ON(!channel->qsdev);
 922
 923         dev = &channel->qsdev->dev;
 924
 925         device_unregister(dev);
 926         of_node_put(dev->of_node);
 927         put_device(dev);
 928 }
 929
 930 /**
 931  * qcom_smd_driver_register - register a smd driver
 932  * @qsdrv:      qcom_smd_driver struct
 933  */
 934 int qcom_smd_driver_register(struct qcom_smd_driver *qsdrv)
 935 {
 936         qsdrv->driver.bus = &qcom_smd_bus;
 937         return driver_register(&qsdrv->driver);
 938 }
 939 EXPORT_SYMBOL(qcom_smd_driver_register);
 940
 941 /**
 942  * qcom_smd_driver_unregister - unregister a smd driver
 943  * @qsdrv:      qcom_smd_driver struct
 944  */
 945 void qcom_smd_driver_unregister(struct qcom_smd_driver *qsdrv)
 946 {
 947         driver_unregister(&qsdrv->driver);
 948 }
 949 EXPORT_SYMBOL(qcom_smd_driver_unregister);
 950
 951 /*
 952  * Allocate the qcom_smd_channel object for a newly found smd channel,
 953  * retrieving and validating the smem items involved.
 954  */
 955 static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *edge,
 956                                                         unsigned smem_info_item,
 957                                                         unsigned smem_fifo_item,
 958                                                         char *name)
 959 {
 960         struct qcom_smd_channel *channel;
 961         struct qcom_smd *smd = edge->smd;
 962         size_t fifo_size;
 963         size_t info_size;
 964         void *fifo_base;
 965         void *info;
 966         int ret;
 967
 968         channel = devm_kzalloc(smd->dev, sizeof(*channel), GFP_KERNEL);
 969         if (!channel)
 970                 return ERR_PTR(-ENOMEM);
 971
 972         channel->edge = edge;
 973         channel->name = devm_kstrdup(smd->dev, name, GFP_KERNEL);
 974         if (!channel->name)
 975                 return ERR_PTR(-ENOMEM);
 976
 977         mutex_init(&channel->tx_lock);
 978         spin_lock_init(&channel->recv_lock);
 979         init_waitqueue_head(&channel->fblockread_event);
 980
 981         ret = qcom_smem_get(edge->remote_pid, smem_info_item, (void **)&info,
 982                             &info_size);
 983         if (ret)
 984                 goto free_name_and_channel;
 985
 986         /*
 987          * Use the size of the item to figure out which channel info struct to
 988          * use.
 989          */
 990         if (info_size == 2 * sizeof(struct smd_channel_info_word)) {
 991                 channel->tx_info_word = info;
 992                 channel->rx_info_word = info + sizeof(struct smd_channel_info_word);
 993         } else if (info_size == 2 * sizeof(struct smd_channel_info)) {
 994                 channel->tx_info = info;
 995                 channel->rx_info = info + sizeof(struct smd_channel_info);
 996         } else {
 997                 dev_err(smd->dev,
 998                         "channel info of size %zu not supported\n", info_size);
 999                 ret = -EINVAL;
1000                 goto free_name_and_channel;
1001         }
1002
1003         ret = qcom_smem_get(edge->remote_pid, smem_fifo_item, &fifo_base,
1004                             &fifo_size);
1005         if (ret)
1006                 goto free_name_and_channel;
1007
1008         /* The channel consist of a rx and tx fifo of equal size */
1009         fifo_size /= 2;
1010
1011         dev_dbg(smd->dev, "new channel '%s' info-size: %zu fifo-size: %zu\n",
1012                           name, info_size, fifo_size);
1013
1014         channel->tx_fifo = fifo_base;
1015         channel->rx_fifo = fifo_base + fifo_size;
1016         channel->fifo_size = fifo_size;
1017
1018         qcom_smd_channel_reset(channel);
1019
1020         return channel;
1021
1022 free_name_and_channel:
1023         devm_kfree(smd->dev, channel->name);
1024         devm_kfree(smd->dev, channel);
1025
1026         return ERR_PTR(ret);
1027 }
1028
1029 /*
1030  * Scans the allocation table for any newly allocated channels, calls
1031  * qcom_smd_create_channel() to create representations of these and add
1032  * them to the edge's list of channels.
1033  */
1034 static void qcom_discover_channels(struct qcom_smd_edge *edge)
1035 {
1036         struct qcom_smd_alloc_entry *alloc_tbl;
1037         struct qcom_smd_alloc_entry *entry;
1038         struct qcom_smd_channel *channel;
1039         struct qcom_smd *smd = edge->smd;
1040         unsigned long flags;
1041         unsigned fifo_id;
1042         unsigned info_id;
1043         int ret;
1044         int tbl;
1045         int i;
1046
1047         for (tbl = 0; tbl < SMD_ALLOC_TBL_COUNT; tbl++) {
1048                 ret = qcom_smem_get(edge->remote_pid,
1049                                     smem_items[tbl].alloc_tbl_id,
1050                                     (void **)&alloc_tbl,
1051                                     NULL);
1052                 if (ret < 0)
1053                         continue;
1054
1055                 for (i = 0; i < SMD_ALLOC_TBL_SIZE; i++) {
1056                         entry = &alloc_tbl[i];
1057                         if (test_bit(i, edge->allocated[tbl]))
1058                                 continue;
1059
1060                         if (entry->ref_count == 0)
1061                                 continue;
1062
1063                         if (!entry->name[0])
1064                                 continue;
1065
1066                         if (!(entry->flags & SMD_CHANNEL_FLAGS_PACKET))
1067                                 continue;
1068
1069                         if ((entry->flags & SMD_CHANNEL_FLAGS_EDGE_MASK) != edge->edge_id)
1070                                 continue;
1071
1072                         info_id = smem_items[tbl].info_base_id + entry->cid;
1073                         fifo_id = smem_items[tbl].fifo_base_id + entry->cid;
1074
1075                         channel = qcom_smd_create_channel(edge, info_id, fifo_id, entry->name);
1076                         if (IS_ERR(channel))
1077                                 continue;
1078
1079                         spin_lock_irqsave(&edge->channels_lock, flags);
1080                         list_add(&channel->list, &edge->channels);
1081                         spin_unlock_irqrestore(&edge->channels_lock, flags);
1082
1083                         dev_dbg(smd->dev, "new channel found: '%s'\n", channel->name);
1084                         set_bit(i, edge->allocated[tbl]);
1085                 }
1086         }
1087
1088         schedule_work(&edge->work);
1089 }
1090
1091 /*
1092  * This per edge worker scans smem for any new channels and register these. It
1093  * then scans all registered channels for state changes that should be handled
1094  * by creating or destroying smd client devices for the registered channels.
1095  *
1096  * LOCKING: edge->channels_lock is not needed to be held during the traversal
1097  * of the channels list as it's done synchronously with the only writer.
1098  */
1099 static void qcom_channel_state_worker(struct work_struct *work)
1100 {
1101         struct qcom_smd_channel *channel;
1102         struct qcom_smd_edge *edge = container_of(work,
1103                                                   struct qcom_smd_edge,
1104                                                   work);
1105         unsigned remote_state;
1106
1107         /*
1108          * Rescan smem if we have reason to belive that there are new channels.
1109          */
1110         if (edge->need_rescan) {
1111                 edge->need_rescan = false;
1112                 qcom_discover_channels(edge);
1113         }
1114
1115         /*
1116          * Register a device for any closed channel where the remote processor
1117          * is showing interest in opening the channel.
1118          */
1119         list_for_each_entry(channel, &edge->channels, list) {
1120                 if (channel->state != SMD_CHANNEL_CLOSED)
1121                         continue;
1122
1123                 remote_state = GET_RX_CHANNEL_INFO(channel, state);
1124                 if (remote_state != SMD_CHANNEL_OPENING &&
1125                     remote_state != SMD_CHANNEL_OPENED)
1126                         continue;
1127
1128                 qcom_smd_create_device(channel);
1129         }
1130
1131         /*
1132          * Unregister the device for any channel that is opened where the
1133          * remote processor is closing the channel.
1134          */
1135         list_for_each_entry(channel, &edge->channels, list) {
1136                 if (channel->state != SMD_CHANNEL_OPENING &&
1137                     channel->state != SMD_CHANNEL_OPENED)
1138                         continue;
1139
1140                 remote_state = GET_RX_CHANNEL_INFO(channel, state);
1141                 if (remote_state == SMD_CHANNEL_OPENING ||
1142                     remote_state == SMD_CHANNEL_OPENED)
1143                         continue;
1144
1145                 qcom_smd_destroy_device(channel);
1146         }
1147 }
1148
1149 /*
1150  * Parses an of_node describing an edge.
1151  */
1152 static int qcom_smd_parse_edge(struct device *dev,
1153                                struct device_node *node,
1154                                struct qcom_smd_edge *edge)
1155 {
1156         struct device_node *syscon_np;
1157         const char *key;
1158         int irq;
1159         int ret;
1160
1161         INIT_LIST_HEAD(&edge->channels);
1162         spin_lock_init(&edge->channels_lock);
1163
1164         INIT_WORK(&edge->work, qcom_channel_state_worker);
1165
1166         edge->of_node = of_node_get(node);
1167
1168         irq = irq_of_parse_and_map(node, 0);
1169         if (irq < 0) {
1170                 dev_err(dev, "required smd interrupt missing\n");
1171                 return -EINVAL;
1172         }
1173
1174         ret = devm_request_irq(dev, irq,
1175                                qcom_smd_edge_intr, IRQF_TRIGGER_RISING,
1176                                node->name, edge);
1177         if (ret) {
1178                 dev_err(dev, "failed to request smd irq\n");
1179                 return ret;
1180         }
1181
1182         edge->irq = irq;
1183
1184         key = "qcom,smd-edge";
1185         ret = of_property_read_u32(node, key, &edge->edge_id);
1186         if (ret) {
1187                 dev_err(dev, "edge missing %s property\n", key);
1188                 return -EINVAL;
1189         }
1190
1191         edge->remote_pid = QCOM_SMEM_HOST_ANY;
1192         key = "qcom,remote-pid";
1193         of_property_read_u32(node, key, &edge->remote_pid);
1194
1195         syscon_np = of_parse_phandle(node, "qcom,ipc", 0);
1196         if (!syscon_np) {
1197                 dev_err(dev, "no qcom,ipc node\n");
1198                 return -ENODEV;
1199         }
1200
1201         edge->ipc_regmap = syscon_node_to_regmap(syscon_np);
1202         if (IS_ERR(edge->ipc_regmap))
1203                 return PTR_ERR(edge->ipc_regmap);
1204
1205         key = "qcom,ipc";
1206         ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset);
1207         if (ret < 0) {
1208                 dev_err(dev, "no offset in %s\n", key);
1209                 return -EINVAL;
1210         }
1211
1212         ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit);
1213         if (ret < 0) {
1214                 dev_err(dev, "no bit in %s\n", key);
1215                 return -EINVAL;
1216         }
1217
1218         return 0;
1219 }
1220
1221 static int qcom_smd_probe(struct platform_device *pdev)
1222 {
1223         struct qcom_smd_edge *edge;
1224         struct device_node *node;
1225         struct qcom_smd *smd;
1226         size_t array_size;
1227         int num_edges;
1228         int ret;
1229         int i = 0;
1230
1231         /* Wait for smem */
1232         ret = qcom_smem_get(QCOM_SMEM_HOST_ANY, smem_items[0].alloc_tbl_id, NULL, NULL);
1233         if (ret == -EPROBE_DEFER)
1234                 return ret;
1235
1236         num_edges = of_get_available_child_count(pdev->dev.of_node);
1237         array_size = sizeof(*smd) + num_edges * sizeof(struct qcom_smd_edge);
1238         smd = devm_kzalloc(&pdev->dev, array_size, GFP_KERNEL);
1239         if (!smd)
1240                 return -ENOMEM;
1241         smd->dev = &pdev->dev;
1242
1243         smd->num_edges = num_edges;
1244         for_each_available_child_of_node(pdev->dev.of_node, node) {
1245                 edge = &smd->edges[i++];
1246                 edge->smd = smd;
1247
1248                 ret = qcom_smd_parse_edge(&pdev->dev, node, edge);
1249                 if (ret)
1250                         continue;
1251
1252                 edge->need_rescan = true;
1253                 schedule_work(&edge->work);
1254         }
1255
1256         platform_set_drvdata(pdev, smd);
1257
1258         return 0;
1259 }
1260
1261 /*
1262  * Shut down all smd clients by making sure that each edge stops processing
1263  * events and scanning for new channels, then call destroy on the devices.
1264  */
1265 static int qcom_smd_remove(struct platform_device *pdev)
1266 {
1267         struct qcom_smd_channel *channel;
1268         struct qcom_smd_edge *edge;
1269         struct qcom_smd *smd = platform_get_drvdata(pdev);
1270         int i;
1271
1272         for (i = 0; i < smd->num_edges; i++) {
1273                 edge = &smd->edges[i];
1274
1275                 disable_irq(edge->irq);
1276                 cancel_work_sync(&edge->work);
1277
1278                 list_for_each_entry(channel, &edge->channels, list) {
1279                         if (!channel->qsdev)
1280                                 continue;
1281
1282                         qcom_smd_destroy_device(channel);
1283                 }
1284         }
1285
1286         return 0;
1287 }
1288
1289 static const struct of_device_id qcom_smd_of_match[] = {
1290         { .compatible = "qcom,smd" },
1291         {}
1292 };
1293 MODULE_DEVICE_TABLE(of, qcom_smd_of_match);
1294
1295 static struct platform_driver qcom_smd_driver = {
1296         .probe = qcom_smd_probe,
1297         .remove = qcom_smd_remove,
1298         .driver = {
1299                 .name = "qcom-smd",
1300                 .of_match_table = qcom_smd_of_match,
1301         },
1302 };
1303
1304 static int __init qcom_smd_init(void)
1305 {
1306         int ret;
1307
1308         ret = bus_register(&qcom_smd_bus);
1309         if (ret) {
1310                 pr_err("failed to register smd bus: %d\n", ret);
1311                 return ret;
1312         }
1313
1314         return platform_driver_register(&qcom_smd_driver);
1315 }
1316 postcore_initcall(qcom_smd_init);
1317
1318 static void __exit qcom_smd_exit(void)
1319 {
1320         platform_driver_unregister(&qcom_smd_driver);
1321         bus_unregister(&qcom_smd_bus);
1322 }
1323 module_exit(qcom_smd_exit);
1324
1325 MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
1326 MODULE_DESCRIPTION("Qualcomm Shared Memory Driver");
1327 MODULE_LICENSE("GPL v2");