drivers/gpu/arm/midgard/mali_kbase_tlstream.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <linux/anon_inodes.h>
  19 #include <linux/atomic.h>
  20 #include <linux/file.h>
  21 #include <linux/mutex.h>
  22 #include <linux/poll.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/string.h>
  25 #include <linux/stringify.h>
  26 #include <linux/timer.h>
  27 #include <linux/wait.h>
  28
  29 #include <mali_kbase.h>
  30 #include <mali_kbase_jm.h>
  31 #include <mali_kbase_tlstream.h>
  32
  33 /*****************************************************************************/
  34
  35 /* The version of swtrace protocol used in timeline stream. */
  36 #define SWTRACE_VERSION    3
  37
  38 /* The maximum expected length of string in tracepoint descriptor. */
  39 #define STRLEN_MAX         64 /* bytes */
  40
  41 /* The number of nanoseconds in a second. */
  42 #define NSECS_IN_SEC       1000000000ull /* ns */
  43
  44 /* The period of autoflush checker execution in milliseconds. */
  45 #define AUTOFLUSH_INTERVAL 1000 /* ms */
  46
  47 /* The maximum size of a single packet used by timeline. */
  48 #define PACKET_SIZE        4096 /* bytes */
  49
  50 /* The number of packets used by one timeline stream. */
  51 #define PACKET_COUNT       16
  52
  53 /* The number of bytes reserved for packet header.
  54  * These value must be defined according to MIPE documentation. */
  55 #define PACKET_HEADER_SIZE 8 /* bytes */
  56
  57 /* The number of bytes reserved for packet sequence number.
  58  * These value must be defined according to MIPE documentation. */
  59 #define PACKET_NUMBER_SIZE 4 /* bytes */
  60
  61 /* Packet header - first word.
  62  * These values must be defined according to MIPE documentation. */
  63 #define PACKET_STREAMID_POS  0
  64 #define PACKET_STREAMID_LEN  8
  65 #define PACKET_RSVD1_POS     (PACKET_STREAMID_POS + PACKET_STREAMID_LEN)
  66 #define PACKET_RSVD1_LEN     8
  67 #define PACKET_TYPE_POS      (PACKET_RSVD1_POS + PACKET_RSVD1_LEN)
  68 #define PACKET_TYPE_LEN      3
  69 #define PACKET_CLASS_POS     (PACKET_TYPE_POS + PACKET_TYPE_LEN)
  70 #define PACKET_CLASS_LEN     7
  71 #define PACKET_FAMILY_POS    (PACKET_CLASS_POS + PACKET_CLASS_LEN)
  72 #define PACKET_FAMILY_LEN    6
  73
  74 /* Packet header - second word
  75  * These values must be defined according to MIPE documentation. */
  76 #define PACKET_LENGTH_POS    0
  77 #define PACKET_LENGTH_LEN    24
  78 #define PACKET_SEQBIT_POS    (PACKET_LENGTH_POS + PACKET_LENGTH_LEN)
  79 #define PACKET_SEQBIT_LEN    1
  80 #define PACKET_RSVD2_POS     (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN)
  81 #define PACKET_RSVD2_LEN     7
  82
  83 /* Types of streams generated by timeline.
  84  * Order is significant! Header streams must precede respective body streams. */
  85 enum tl_stream_type {
  86         TL_STREAM_TYPE_OBJ_HEADER,
  87         TL_STREAM_TYPE_OBJ_SUMMARY,
  88         TL_STREAM_TYPE_OBJ,
  89         TL_STREAM_TYPE_AUX_HEADER,
  90         TL_STREAM_TYPE_AUX,
  91
  92         TL_STREAM_TYPE_COUNT
  93 };
  94
  95 /* Timeline packet family ids.
  96  * Values are significant! Check MIPE documentation. */
  97 enum tl_packet_family {
  98         TL_PACKET_FAMILY_CTRL = 0, /* control packets */
  99         TL_PACKET_FAMILY_TL   = 1, /* timeline packets */
 100
 101         TL_PACKET_FAMILY_COUNT
 102 };
 103
 104 /* Packet classes used in timeline streams.
 105  * Values are significant! Check MIPE documentation. */
 106 enum tl_packet_class {
 107         TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */
 108         TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */
 109 };
 110
 111 /* Packet types used in timeline streams.
 112  * Values are significant! Check MIPE documentation. */
 113 enum tl_packet_type {
 114         TL_PACKET_TYPE_HEADER  = 0, /* stream's header/directory */
 115         TL_PACKET_TYPE_BODY    = 1, /* stream's body */
 116         TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */
 117 };
 118
 119 /* Message ids of trace events that are recorded in the timeline stream. */
 120 enum tl_msg_id_obj {
 121         /* Timeline object events. */
 122         KBASE_TL_NEW_CTX,
 123         KBASE_TL_NEW_GPU,
 124         KBASE_TL_NEW_LPU,
 125         KBASE_TL_NEW_ATOM,
 126         KBASE_TL_NEW_AS,
 127         KBASE_TL_DEL_CTX,
 128         KBASE_TL_DEL_ATOM,
 129         KBASE_TL_LIFELINK_LPU_GPU,
 130         KBASE_TL_LIFELINK_AS_GPU,
 131         KBASE_TL_RET_CTX_LPU,
 132         KBASE_TL_RET_ATOM_CTX,
 133         KBASE_TL_RET_ATOM_LPU,
 134         KBASE_TL_NRET_CTX_LPU,
 135         KBASE_TL_NRET_ATOM_CTX,
 136         KBASE_TL_NRET_ATOM_LPU,
 137         KBASE_TL_RET_AS_CTX,
 138         KBASE_TL_NRET_AS_CTX,
 139         KBASE_TL_RET_ATOM_AS,
 140         KBASE_TL_NRET_ATOM_AS,
 141         KBASE_TL_DEP_ATOM_ATOM,
 142         KBASE_TL_NDEP_ATOM_ATOM,
 143         KBASE_TL_RDEP_ATOM_ATOM,
 144         KBASE_TL_ATTRIB_ATOM_CONFIG,
 145         KBASE_TL_ATTRIB_AS_CONFIG,
 146
 147         /* Job dump specific events. */
 148         KBASE_JD_GPU_SOFT_RESET
 149 };
 150
 151 /* Message ids of trace events that are recorded in the auxiliary stream. */
 152 enum tl_msg_id_aux {
 153         KBASE_AUX_PM_STATE,
 154         KBASE_AUX_ISSUE_JOB_SOFTSTOP,
 155         KBASE_AUX_JOB_SOFTSTOP,
 156         KBASE_AUX_JOB_SOFTSTOP_EX,
 157         KBASE_AUX_PAGEFAULT,
 158         KBASE_AUX_PAGESALLOC
 159 };
 160
 161 /*****************************************************************************/
 162
 163 /**
 164  * struct tl_stream - timeline stream structure
 165  * @lock: message order lock
 166  * @buffer: array of buffers
 167  * @wbi: write buffer index
 168  * @rbi: read buffer index
 169  * @numbered: if non-zero stream's packets are sequentially numbered
 170  * @autoflush_counter: counter tracking stream's autoflush state
 171  *
 172  * This structure holds information needed to construct proper packets in the
 173  * timeline stream. Each message in sequence must bear timestamp that is greater
 174  * to one in previous message in the same stream. For this reason lock is held
 175  * throughout the process of message creation. Each stream contains set of
 176  * buffers. Each buffer will hold one MIPE packet. In case there is no free
 177  * space required to store incoming message the oldest buffer is discarded.
 178  * Each packet in timeline body stream has sequence number embedded (this value
 179  * must increment monotonically and is used by packets receiver to discover
 180  * buffer overflows.
 181  * Autoflush counter is set to negative number when there is no data pending
 182  * for flush and it is set to zero on every update of the buffer. Autoflush
 183  * timer will increment the counter by one on every expiry. In case there will
 184  * be no activity on the buffer during two consecutive timer expiries, stream
 185  * buffer will be flushed.
 186  */
 187 struct tl_stream {
 188         spinlock_t lock;
 189
 190         struct {
 191                 atomic_t size;              /* number of bytes in buffer */
 192                 char     data[PACKET_SIZE]; /* buffer's data */
 193         } buffer[PACKET_COUNT];
 194
 195         atomic_t wbi;
 196         atomic_t rbi;
 197
 198         int      numbered;
 199         atomic_t autoflush_counter;
 200 };
 201
 202 /**
 203  * struct tp_desc - tracepoint message descriptor structure
 204  * @id:        tracepoint ID identifying message in stream
 205  * @id_str:    human readable version of tracepoint ID
 206  * @name:      tracepoint description
 207  * @arg_types: tracepoint's arguments types declaration
 208  * @arg_names: comma separated list of tracepoint's arguments names
 209  */
 210 struct tp_desc {
 211         u32        id;
 212         const char *id_str;
 213         const char *name;
 214         const char *arg_types;
 215         const char *arg_names;
 216 };
 217
 218 /*****************************************************************************/
 219
 220 /* Configuration of timeline streams generated by kernel.
 221  * Kernel emit only streams containing either timeline object events or
 222  * auxiliary events. All streams have stream id value of 1 (as opposed to user
 223  * space streams that have value of 0). */
 224 static const struct {
 225         enum tl_packet_family pkt_family;
 226         enum tl_packet_class  pkt_class;
 227         enum tl_packet_type   pkt_type;
 228         unsigned int          stream_id;
 229 } tl_stream_cfg[TL_STREAM_TYPE_COUNT] = {
 230         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER,  1},
 231         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1},
 232         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY,    1},
 233         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER,  1},
 234         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY,    1}
 235 };
 236
 237 /* The timeline streams generated by kernel. */
 238 static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT];
 239
 240 /* Autoflush timer. */
 241 static struct timer_list autoflush_timer;
 242
 243 /* If non-zero autoflush timer is active. */
 244 static atomic_t autoflush_timer_active;
 245
 246 /* Reader lock. Only one reader is allowed to have access to the timeline
 247  * streams at any given time. */
 248 static DEFINE_MUTEX(tl_reader_lock);
 249
 250 /* Timeline stream event queue. */
 251 static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue);
 252
 253 /* The timeline stream file operations functions. */
 254 static ssize_t kbasep_tlstream_read(
 255                 struct file *filp,
 256                 char __user *buffer,
 257                 size_t      size,
 258                 loff_t      *f_pos);
 259 static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait);
 260 static int kbasep_tlstream_release(struct inode *inode, struct file *filp);
 261
 262 /* The timeline stream file operations structure. */
 263 static const struct file_operations kbasep_tlstream_fops = {
 264         .release = kbasep_tlstream_release,
 265         .read    = kbasep_tlstream_read,
 266         .poll    = kbasep_tlstream_poll,
 267 };
 268
 269 /* Descriptors of timeline messages transmitted in object events stream. */
 270 static const struct tp_desc tp_desc_obj[] = {
 271         {
 272                 KBASE_TL_NEW_CTX,
 273                 __stringify(KBASE_TL_NEW_CTX),
 274                 "object ctx is created",
 275                 "@pII",
 276                 "ctx,ctx_nr,tgid"
 277         },
 278         {
 279                 KBASE_TL_NEW_GPU,
 280                 __stringify(KBASE_TL_NEW_GPU),
 281                 "object gpu is created",
 282                 "@pII",
 283                 "gpu,gpu_id,core_count"
 284         },
 285         {
 286                 KBASE_TL_NEW_LPU,
 287                 __stringify(KBASE_TL_NEW_LPU),
 288                 "object lpu is created",
 289                 "@pII",
 290                 "lpu,lpu_nr,lpu_fn"
 291         },
 292         {
 293                 KBASE_TL_NEW_ATOM,
 294                 __stringify(KBASE_TL_NEW_ATOM),
 295                 "object atom is created",
 296                 "@pI",
 297                 "atom,atom_nr"
 298         },
 299         {
 300                 KBASE_TL_NEW_AS,
 301                 __stringify(KBASE_TL_NEW_AS),
 302                 "address space object is created",
 303                 "@pI",
 304                 "address_space,as_nr"
 305         },
 306         {
 307                 KBASE_TL_DEL_CTX,
 308                 __stringify(KBASE_TL_DEL_CTX),
 309                 "context is destroyed",
 310                 "@p",
 311                 "ctx"
 312         },
 313         {
 314                 KBASE_TL_DEL_ATOM,
 315                 __stringify(KBASE_TL_DEL_ATOM),
 316                 "atom is destroyed",
 317                 "@p",
 318                 "atom"
 319         },
 320         {
 321                 KBASE_TL_LIFELINK_LPU_GPU,
 322                 __stringify(KBASE_TL_LIFELINK_LPU_GPU),
 323                 "lpu is deleted with gpu",
 324                 "@pp",
 325                 "lpu,gpu"
 326         },
 327         {
 328                 KBASE_TL_LIFELINK_AS_GPU,
 329                 __stringify(KBASE_TL_LIFELINK_AS_GPU),
 330                 "address space is deleted with gpu",
 331                 "@pp",
 332                 "address_space,gpu"
 333         },
 334         {
 335                 KBASE_TL_RET_CTX_LPU,
 336                 __stringify(KBASE_TL_RET_CTX_LPU),
 337                 "context is retained by lpu",
 338                 "@pp",
 339                 "ctx,lpu"
 340         },
 341         {
 342                 KBASE_TL_RET_ATOM_CTX,
 343                 __stringify(KBASE_TL_RET_ATOM_CTX),
 344                 "atom is retained by context",
 345                 "@pp",
 346                 "atom,ctx"
 347         },
 348         {
 349                 KBASE_TL_RET_ATOM_LPU,
 350                 __stringify(KBASE_TL_RET_ATOM_LPU),
 351                 "atom is retained by lpu",
 352                 "@pps",
 353                 "atom,lpu,attrib_match_list"
 354         },
 355         {
 356                 KBASE_TL_NRET_CTX_LPU,
 357                 __stringify(KBASE_TL_NRET_CTX_LPU),
 358                 "context is released by lpu",
 359                 "@pp",
 360                 "ctx,lpu"
 361         },
 362         {
 363                 KBASE_TL_NRET_ATOM_CTX,
 364                 __stringify(KBASE_TL_NRET_ATOM_CTX),
 365                 "atom is released by context",
 366                 "@pp",
 367                 "atom,ctx"
 368         },
 369         {
 370                 KBASE_TL_NRET_ATOM_LPU,
 371                 __stringify(KBASE_TL_NRET_ATOM_LPU),
 372                 "atom is released by lpu",
 373                 "@pp",
 374                 "atom,lpu"
 375         },
 376         {
 377                 KBASE_TL_RET_AS_CTX,
 378                 __stringify(KBASE_TL_RET_AS_CTX),
 379                 "address space is retained by context",
 380                 "@pp",
 381                 "address_space,ctx"
 382         },
 383         {
 384                 KBASE_TL_NRET_AS_CTX,
 385                 __stringify(KBASE_TL_NRET_AS_CTX),
 386                 "address space is released by context",
 387                 "@pp",
 388                 "address_space,ctx"
 389         },
 390         {
 391                 KBASE_TL_RET_ATOM_AS,
 392                 __stringify(KBASE_TL_RET_ATOM_AS),
 393                 "atom is retained by address space",
 394                 "@pp",
 395                 "atom,address_space"
 396         },
 397         {
 398                 KBASE_TL_NRET_ATOM_AS,
 399                 __stringify(KBASE_TL_NRET_ATOM_AS),
 400                 "atom is released by address space",
 401                 "@pp",
 402                 "atom,address_space"
 403         },
 404         {
 405                 KBASE_TL_DEP_ATOM_ATOM,
 406                 __stringify(KBASE_TL_DEP_ATOM_ATOM),
 407                 "atom2 depends on atom1",
 408                 "@pp",
 409                 "atom1,atom2"
 410         },
 411         {
 412                 KBASE_TL_NDEP_ATOM_ATOM,
 413                 __stringify(KBASE_TL_NDEP_ATOM_ATOM),
 414                 "atom2 no longer depends on atom1",
 415                 "@pp",
 416                 "atom1,atom2"
 417         },
 418         {
 419                 KBASE_TL_RDEP_ATOM_ATOM,
 420                 __stringify(KBASE_TL_RDEP_ATOM_ATOM),
 421                 "resolved dependecy of atom2 depending on atom1",
 422                 "@pp",
 423                 "atom1,atom2"
 424         },
 425         {
 426                 KBASE_TL_ATTRIB_ATOM_CONFIG,
 427                 __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG),
 428                 "atom job slot attributes",
 429                 "@pLLI",
 430                 "atom,descriptor,affinity,config"
 431         },
 432         {
 433                 KBASE_TL_ATTRIB_AS_CONFIG,
 434                 __stringify(KBASE_TL_ATTRIB_AS_CONFIG),
 435                 "address space attributes",
 436                 "@pLLL",
 437                 "address_space,transtab,memattr,transcfg"
 438         },
 439         {
 440                 KBASE_JD_GPU_SOFT_RESET,
 441                 __stringify(KBASE_JD_GPU_SOFT_RESET),
 442                 "gpu soft reset",
 443                 "@p",
 444                 "gpu"
 445         },
 446 };
 447
 448 /* Descriptors of timeline messages transmitted in auxiliary events stream. */
 449 static const struct tp_desc tp_desc_aux[] = {
 450         {
 451                 KBASE_AUX_PM_STATE,
 452                 __stringify(KBASE_AUX_PM_STATE),
 453                 "PM state",
 454                 "@IL",
 455                 "core_type,core_state_bitset"
 456         },
 457         {
 458                 KBASE_AUX_ISSUE_JOB_SOFTSTOP,
 459                 __stringify(KBASE_AUX_ISSUE_JOB_SOFTSTOP),
 460                 "Issuing job soft stop",
 461                 "@p",
 462                 "atom"
 463         },
 464         {
 465                 KBASE_AUX_JOB_SOFTSTOP,
 466                 __stringify(KBASE_AUX_JOB_SOFTSTOP),
 467                 "Job soft stop",
 468                 "@I",
 469                 "tag_id"
 470         },
 471         {
 472                 KBASE_AUX_JOB_SOFTSTOP_EX,
 473                 __stringify(KBASE_AUX_JOB_SOFTSTOP_EX),
 474                 "Job soft stop, more details",
 475                 "@pI",
 476                 "atom,job_type"
 477         },
 478         {
 479                 KBASE_AUX_PAGEFAULT,
 480                 __stringify(KBASE_AUX_PAGEFAULT),
 481                 "Page fault",
 482                 "@IL",
 483                 "ctx_nr,page_cnt_change"
 484         },
 485         {
 486                 KBASE_AUX_PAGESALLOC,
 487                 __stringify(KBASE_AUX_PAGESALLOC),
 488                 "Total alloc pages change",
 489                 "@IL",
 490                 "ctx_nr,page_cnt"
 491         }
 492 };
 493
 494 #if MALI_UNIT_TEST
 495 /* Number of bytes read by user. */
 496 static atomic_t tlstream_bytes_collected = {0};
 497
 498 /* Number of bytes generated by tracepoint messages. */
 499 static atomic_t tlstream_bytes_generated = {0};
 500 #endif /* MALI_UNIT_TEST */
 501
 502 /*****************************************************************************/
 503
 504 /* Indicator of whether the timeline stream file descriptor is used. */
 505 atomic_t kbase_tlstream_enabled = {0};
 506
 507 /*****************************************************************************/
 508
 509 /**
 510  * kbasep_tlstream_get_timestamp - return timestamp
 511  *
 512  * Function returns timestamp value based on raw monotonic timer. Value will
 513  * wrap around zero in case of overflow.
 514  * Return: timestamp value
 515  */
 516 static u64 kbasep_tlstream_get_timestamp(void)
 517 {
 518         struct timespec ts;
 519         u64             timestamp;
 520
 521         getrawmonotonic(&ts);
 522         timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
 523         return timestamp;
 524 }
 525
 526 /**
 527  * kbasep_tlstream_write_bytes - write data to message buffer
 528  * @buffer: buffer where data will be written
 529  * @pos:    position in the buffer where to place data
 530  * @bytes:  pointer to buffer holding data
 531  * @len:    length of data to be written
 532  *
 533  * Return: updated position in the buffer
 534  */
 535 static size_t kbasep_tlstream_write_bytes(
 536                 char       *buffer,
 537                 size_t     pos,
 538                 const void *bytes,
 539                 size_t     len)
 540 {
 541         KBASE_DEBUG_ASSERT(buffer);
 542         KBASE_DEBUG_ASSERT(bytes);
 543
 544         memcpy(&buffer[pos], bytes, len);
 545
 546         return pos + len;
 547 }
 548
 549 /**
 550  * kbasep_tlstream_write_string - write string to message buffer
 551  * @buffer:         buffer where data will be written
 552  * @pos:            position in the buffer where to place data
 553  * @string:         pointer to buffer holding the source string
 554  * @max_write_size: number of bytes that can be stored in buffer
 555  *
 556  * Return: updated position in the buffer
 557  */
 558 static size_t kbasep_tlstream_write_string(
 559                 char       *buffer,
 560                 size_t     pos,
 561                 const char *string,
 562                 size_t     max_write_size)
 563 {
 564         u32 string_len;
 565
 566         KBASE_DEBUG_ASSERT(buffer);
 567         KBASE_DEBUG_ASSERT(string);
 568         /* Timeline string consists of at least string length and nul
 569          * terminator. */
 570         KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char));
 571         max_write_size -= sizeof(string_len);
 572
 573         string_len = strlcpy(
 574                         &buffer[pos + sizeof(string_len)],
 575                         string,
 576                         max_write_size);
 577         string_len += sizeof(char);
 578
 579         /* Make sure that the source string fit into the buffer. */
 580         KBASE_DEBUG_ASSERT(string_len <= max_write_size);
 581
 582         /* Update string length. */
 583         memcpy(&buffer[pos], &string_len, sizeof(string_len));
 584
 585         return pos + sizeof(string_len) + string_len;
 586 }
 587
 588 /**
 589  * kbasep_tlstream_write_timestamp - write timestamp to message buffer
 590  * @buffer: buffer where data will be written
 591  * @pos:    position in the buffer where to place data
 592  *
 593  * Return: updated position in the buffer
 594  */
 595 static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos)
 596 {
 597         u64 timestamp = kbasep_tlstream_get_timestamp();
 598
 599         return kbasep_tlstream_write_bytes(
 600                         buffer, pos,
 601                         &timestamp, sizeof(timestamp));
 602 }
 603
 604 /**
 605  * kbasep_tlstream_put_bits - put bits in a word
 606  * @word:   pointer to the words being modified
 607  * @value:  value that shall be written to given position
 608  * @bitpos: position where value shall be written (in bits)
 609  * @bitlen: length of value (in bits)
 610  */
 611 static void kbasep_tlstream_put_bits(
 612                 u32          *word,
 613                 u32          value,
 614                 unsigned int bitpos,
 615                 unsigned int bitlen)
 616 {
 617         const u32 mask = ((1 << bitlen) - 1) << bitpos;
 618
 619         KBASE_DEBUG_ASSERT(word);
 620         KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen));
 621         KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32);
 622
 623         *word &= ~mask;
 624         *word |= ((value << bitpos) & mask);
 625 }
 626
 627 /**
 628  * kbasep_tlstream_packet_header_setup - setup the packet header
 629  * @buffer:     pointer to the buffer
 630  * @pkt_family: packet's family
 631  * @pkt_type:   packet's type
 632  * @pkt_class:  packet's class
 633  * @stream_id:  stream id
 634  * @numbered:   non-zero if this stream is numbered
 635  *
 636  * Function sets up immutable part of packet header in the given buffer.
 637  */
 638 static void kbasep_tlstream_packet_header_setup(
 639                 char                  *buffer,
 640                 enum tl_packet_family pkt_family,
 641                 enum tl_packet_class  pkt_class,
 642                 enum tl_packet_type   pkt_type,
 643                 unsigned int          stream_id,
 644                 int                   numbered)
 645 {
 646         u32 word0 = 0;
 647         u32 word1 = 0;
 648
 649         KBASE_DEBUG_ASSERT(buffer);
 650         KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL);
 651         KBASE_DEBUG_ASSERT(
 652                         (pkt_type == TL_PACKET_TYPE_HEADER)  ||
 653                         (pkt_type == TL_PACKET_TYPE_SUMMARY) ||
 654                         (pkt_type == TL_PACKET_TYPE_BODY));
 655         KBASE_DEBUG_ASSERT(
 656                         (pkt_class == TL_PACKET_CLASS_OBJ) ||
 657                         (pkt_class == TL_PACKET_CLASS_AUX));
 658
 659         kbasep_tlstream_put_bits(
 660                         &word0, pkt_family,
 661                         PACKET_FAMILY_POS, PACKET_FAMILY_LEN);
 662         kbasep_tlstream_put_bits(
 663                         &word0, pkt_class,
 664                         PACKET_CLASS_POS, PACKET_CLASS_LEN);
 665         kbasep_tlstream_put_bits(
 666                         &word0, pkt_type,
 667                         PACKET_TYPE_POS, PACKET_TYPE_LEN);
 668         kbasep_tlstream_put_bits(
 669                         &word0, stream_id,
 670                         PACKET_STREAMID_POS, PACKET_STREAMID_LEN);
 671
 672         if (numbered)
 673                 kbasep_tlstream_put_bits(
 674                                 &word1, 1,
 675                                 PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN);
 676
 677         memcpy(&buffer[0],             &word0, sizeof(word0));
 678         memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
 679 }
 680
 681 /**
 682  * kbasep_tlstream_packet_header_update - update the packet header
 683  * @buffer:    pointer to the buffer
 684  * @data_size: amount of data carried in this packet
 685  *
 686  * Function updates mutable part of packet header in the given buffer.
 687  * Note that value of data_size must not including size of the header.
 688  */
 689 static void kbasep_tlstream_packet_header_update(
 690                 char   *buffer,
 691                 size_t data_size)
 692 {
 693         u32 word0;
 694         u32 word1;
 695
 696         KBASE_DEBUG_ASSERT(buffer);
 697         CSTD_UNUSED(word0);
 698
 699         memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1));
 700
 701         kbasep_tlstream_put_bits(
 702                         &word1, data_size,
 703                         PACKET_LENGTH_POS, PACKET_LENGTH_LEN);
 704
 705         memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
 706 }
 707
 708 /**
 709  * kbasep_tlstream_packet_number_update - update the packet number
 710  * @buffer:  pointer to the buffer
 711  * @counter: value of packet counter for this packet's stream
 712  *
 713  * Function updates packet number embedded within the packet placed in the
 714  * given buffer.
 715  */
 716 static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter)
 717 {
 718         KBASE_DEBUG_ASSERT(buffer);
 719
 720         memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter));
 721 }
 722
 723 /**
 724  * kbasep_timeline_stream_reset - reset stream
 725  * @stream:  pointer to the stream structure
 726  *
 727  * Function discards all pending messages and resets packet counters.
 728  */
 729 static void kbasep_timeline_stream_reset(struct tl_stream *stream)
 730 {
 731         unsigned int i;
 732
 733         for (i = 0; i < PACKET_COUNT; i++) {
 734                 if (stream->numbered)
 735                         atomic_set(
 736                                         &stream->buffer[i].size,
 737                                         PACKET_HEADER_SIZE +
 738                                         PACKET_NUMBER_SIZE);
 739                 else
 740                         atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE);
 741         }
 742
 743         atomic_set(&stream->wbi, 0);
 744         atomic_set(&stream->rbi, 0);
 745 }
 746
 747 /**
 748  * kbasep_timeline_stream_init - initialize timeline stream
 749  * @stream:      pointer to the stream structure
 750  * @stream_type: stream type
 751  */
 752 static void kbasep_timeline_stream_init(
 753                 struct tl_stream    *stream,
 754                 enum tl_stream_type stream_type)
 755 {
 756         unsigned int i;
 757
 758         KBASE_DEBUG_ASSERT(stream);
 759         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 760
 761         spin_lock_init(&stream->lock);
 762
 763         /* All packets carrying tracepoints shall be numbered. */
 764         if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type)
 765                 stream->numbered = 1;
 766         else
 767                 stream->numbered = 0;
 768
 769         for (i = 0; i < PACKET_COUNT; i++)
 770                 kbasep_tlstream_packet_header_setup(
 771                                 stream->buffer[i].data,
 772                                 tl_stream_cfg[stream_type].pkt_family,
 773                                 tl_stream_cfg[stream_type].pkt_class,
 774                                 tl_stream_cfg[stream_type].pkt_type,
 775                                 tl_stream_cfg[stream_type].stream_id,
 776                                 stream->numbered);
 777
 778         kbasep_timeline_stream_reset(tl_stream[stream_type]);
 779 }
 780
 781 /**
 782  * kbasep_timeline_stream_term - terminate timeline stream
 783  * @stream: pointer to the stream structure
 784  */
 785 static void kbasep_timeline_stream_term(struct tl_stream *stream)
 786 {
 787         KBASE_DEBUG_ASSERT(stream);
 788 }
 789
 790 /**
 791  * kbasep_tlstream_msgbuf_submit - submit packet to the user space
 792  * @stream:     pointer to the stream structure
 793  * @wb_idx_raw: write buffer index
 794  * @wb_size:    length of data stored in current buffer
 795  *
 796  * Function updates currently written buffer with packet header. Then write
 797  * index is incremented and buffer is handled to user space. Parameters
 798  * of new buffer are returned using provided arguments.
 799  *
 800  * Return: length of data in new buffer
 801  *
 802  * Warning:  User must update the stream structure with returned value.
 803  */
 804 static size_t kbasep_tlstream_msgbuf_submit(
 805                 struct tl_stream *stream,
 806                 unsigned int      wb_idx_raw,
 807                 unsigned int      wb_size)
 808 {
 809         unsigned int rb_idx_raw = atomic_read(&stream->rbi);
 810         unsigned int wb_idx = wb_idx_raw % PACKET_COUNT;
 811
 812         /* Set stream as flushed. */
 813         atomic_set(&stream->autoflush_counter, -1);
 814
 815         kbasep_tlstream_packet_header_update(
 816                         stream->buffer[wb_idx].data,
 817                         wb_size - PACKET_HEADER_SIZE);
 818
 819         if (stream->numbered)
 820                 kbasep_tlstream_packet_number_update(
 821                                 stream->buffer[wb_idx].data,
 822                                 wb_idx_raw);
 823
 824         /* Increasing write buffer index will expose this packet to the reader.
 825          * As stream->lock is not taken on reader side we must make sure memory
 826          * is updated correctly before this will happen. */
 827         smp_wmb();
 828         wb_idx_raw++;
 829         atomic_set(&stream->wbi, wb_idx_raw);
 830
 831         /* Inform user that packets are ready for reading. */
 832         wake_up_interruptible(&tl_event_queue);
 833
 834         /* Detect and mark overflow in this stream. */
 835         if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) {
 836                 /* Reader side depends on this increment to correctly handle
 837                  * overflows. The value shall be updated only if it was not
 838                  * modified by the reader. The data holding buffer will not be
 839                  * updated before stream->lock is released, however size of the
 840                  * buffer will. Make sure this increment is globally visible
 841                  * before information about selected write buffer size. */
 842                 atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1);
 843         }
 844
 845         wb_size = PACKET_HEADER_SIZE;
 846         if (stream->numbered)
 847                 wb_size += PACKET_NUMBER_SIZE;
 848
 849         return wb_size;
 850 }
 851
 852 /**
 853  * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer
 854  * @stream_type: type of the stream that shall be locked
 855  * @msg_size:    message size
 856  * @flags:       pointer to store flags passed back on stream release
 857  *
 858  * Function will lock the stream and reserve the number of bytes requested
 859  * in msg_size for the user.
 860  *
 861  * Return: pointer to the buffer where message can be stored
 862  *
 863  * Warning: Stream must be released with kbasep_tlstream_msgbuf_release().
 864  *          Only atomic operations are allowed while stream is locked
 865  *          (i.e. do not use any operation that may sleep).
 866  */
 867 static char *kbasep_tlstream_msgbuf_acquire(
 868                 enum tl_stream_type stream_type,
 869                 size_t              msg_size,
 870                 unsigned long       *flags) __acquires(&stream->lock)
 871 {
 872         struct tl_stream *stream;
 873         unsigned int     wb_idx_raw;
 874         unsigned int     wb_idx;
 875         size_t           wb_size;
 876
 877         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 878         KBASE_DEBUG_ASSERT(
 879                         PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >=
 880                         msg_size);
 881
 882         stream = tl_stream[stream_type];
 883
 884         spin_lock_irqsave(&stream->lock, *flags);
 885
 886         wb_idx_raw = atomic_read(&stream->wbi);
 887         wb_idx     = wb_idx_raw % PACKET_COUNT;
 888         wb_size    = atomic_read(&stream->buffer[wb_idx].size);
 889
 890         /* Select next buffer if data will not fit into current one. */
 891         if (PACKET_SIZE < wb_size + msg_size) {
 892                 wb_size = kbasep_tlstream_msgbuf_submit(
 893                                 stream, wb_idx_raw, wb_size);
 894                 wb_idx  = (wb_idx_raw + 1) % PACKET_COUNT;
 895         }
 896
 897         /* Reserve space in selected buffer. */
 898         atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size);
 899
 900 #if MALI_UNIT_TEST
 901         atomic_add(msg_size, &tlstream_bytes_generated);
 902 #endif /* MALI_UNIT_TEST */
 903
 904         return &stream->buffer[wb_idx].data[wb_size];
 905 }
 906
 907 /**
 908  * kbasep_tlstream_msgbuf_release - unlock selected stream
 909  * @stream_type:  type of the stream that shall be locked
 910  * @flags:        value obtained during stream acquire
 911  *
 912  * Function releases stream that has been previously locked with a call to
 913  * kbasep_tlstream_msgbuf_acquire().
 914  */
 915 static void kbasep_tlstream_msgbuf_release(
 916                 enum tl_stream_type stream_type,
 917                 unsigned long       flags) __releases(&stream->lock)
 918 {
 919         struct tl_stream *stream;
 920
 921         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 922
 923         stream = tl_stream[stream_type];
 924
 925         /* Mark stream as containing unflushed data. */
 926         atomic_set(&stream->autoflush_counter, 0);
 927
 928         spin_unlock_irqrestore(&stream->lock, flags);
 929 }
 930
 931 /*****************************************************************************/
 932
 933 /**
 934  * kbasep_tlstream_flush_stream - flush stream
 935  * @stype:  type of stream to be flushed
 936  *
 937  * Flush pending data in timeline stream.
 938  */
 939 static void kbasep_tlstream_flush_stream(enum tl_stream_type stype)
 940 {
 941         struct tl_stream *stream = tl_stream[stype];
 942         unsigned long    flags;
 943         unsigned int     wb_idx_raw;
 944         unsigned int     wb_idx;
 945         size_t           wb_size;
 946         size_t           min_size = PACKET_HEADER_SIZE;
 947
 948         if (stream->numbered)
 949                 min_size += PACKET_NUMBER_SIZE;
 950
 951         spin_lock_irqsave(&stream->lock, flags);
 952
 953         wb_idx_raw = atomic_read(&stream->wbi);
 954         wb_idx     = wb_idx_raw % PACKET_COUNT;
 955         wb_size    = atomic_read(&stream->buffer[wb_idx].size);
 956
 957         if (wb_size > min_size) {
 958                 wb_size = kbasep_tlstream_msgbuf_submit(
 959                                 stream, wb_idx_raw, wb_size);
 960                 wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
 961                 atomic_set(&stream->buffer[wb_idx].size, wb_size);
 962         }
 963         spin_unlock_irqrestore(&stream->lock, flags);
 964 }
 965
 966 /**
 967  * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback
 968  * @data:  unused
 969  *
 970  * Timer is executed periodically to check if any of the stream contains
 971  * buffer ready to be submitted to user space.
 972  */
 973 static void kbasep_tlstream_autoflush_timer_callback(unsigned long data)
 974 {
 975         enum tl_stream_type stype;
 976         int                 rcode;
 977
 978         CSTD_UNUSED(data);
 979
 980         for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) {
 981                 struct tl_stream *stream = tl_stream[stype];
 982                 unsigned long    flags;
 983                 unsigned int     wb_idx_raw;
 984                 unsigned int     wb_idx;
 985                 size_t           wb_size;
 986                 size_t           min_size = PACKET_HEADER_SIZE;
 987
 988                 int af_cnt = atomic_read(&stream->autoflush_counter);
 989
 990                 /* Check if stream contain unflushed data. */
 991                 if (0 > af_cnt)
 992                         continue;
 993
 994                 /* Check if stream should be flushed now. */
 995                 if (af_cnt != atomic_cmpxchg(
 996                                         &stream->autoflush_counter,
 997                                         af_cnt,
 998                                         af_cnt + 1))
 999                         continue;
1000                 if (!af_cnt)
1001                         continue;
1002
1003                 /* Autoflush this stream. */
1004                 if (stream->numbered)
1005                         min_size += PACKET_NUMBER_SIZE;
1006
1007                 spin_lock_irqsave(&stream->lock, flags);
1008
1009                 wb_idx_raw = atomic_read(&stream->wbi);
1010                 wb_idx     = wb_idx_raw % PACKET_COUNT;
1011                 wb_size    = atomic_read(&stream->buffer[wb_idx].size);
1012
1013                 if (wb_size > min_size) {
1014                         wb_size = kbasep_tlstream_msgbuf_submit(
1015                                         stream, wb_idx_raw, wb_size);
1016                         wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
1017                         atomic_set(&stream->buffer[wb_idx].size,
1018                                         wb_size);
1019                 }
1020                 spin_unlock_irqrestore(&stream->lock, flags);
1021         }
1022
1023         if (atomic_read(&autoflush_timer_active))
1024                 rcode = mod_timer(
1025                                 &autoflush_timer,
1026                                 jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
1027         CSTD_UNUSED(rcode);
1028 }
1029
1030 /**
1031  * kbasep_tlstream_packet_pending - check timeline streams for pending packets
1032  * @stype:      pointer to variable where stream type will be placed
1033  * @rb_idx_raw: pointer to variable where read buffer index will be placed
1034  *
1035  * Function checks all streams for pending packets. It will stop as soon as
1036  * packet ready to be submitted to user space is detected. Variables under
1037  * pointers, passed as the parameters to this function will be updated with
1038  * values pointing to right stream and buffer.
1039  *
1040  * Return: non-zero if any of timeline streams has at last one packet ready
1041  */
1042 static int kbasep_tlstream_packet_pending(
1043                 enum tl_stream_type *stype,
1044                 unsigned int        *rb_idx_raw)
1045 {
1046         int pending = 0;
1047
1048         KBASE_DEBUG_ASSERT(stype);
1049         KBASE_DEBUG_ASSERT(rb_idx_raw);
1050
1051         for (
1052                         *stype = 0;
1053                         (*stype < TL_STREAM_TYPE_COUNT) && !pending;
1054                         (*stype)++) {
1055                 if (NULL != tl_stream[*stype]) {
1056                         *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi);
1057                         /* Read buffer index may be updated by writer in case of
1058                          * overflow. Read and write buffer indexes must be
1059                          * loaded in correct order. */
1060                         smp_rmb();
1061                         if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw)
1062                                 pending = 1;
1063                 }
1064         }
1065         (*stype)--;
1066
1067         return pending;
1068 }
1069
1070 /**
1071  * kbasep_tlstream_read - copy data from streams to buffer provided by user
1072  * @filp:   pointer to file structure (unused)
1073  * @buffer: pointer to the buffer provided by user
1074  * @size:   maximum amount of data that can be stored in the buffer
1075  * @f_pos:  pointer to file offset (unused)
1076  *
1077  * Return: number of bytes stored in the buffer
1078  */
1079 static ssize_t kbasep_tlstream_read(
1080                 struct file *filp,
1081                 char __user *buffer,
1082                 size_t      size,
1083                 loff_t      *f_pos)
1084 {
1085         ssize_t copy_len = 0;
1086
1087         KBASE_DEBUG_ASSERT(filp);
1088         KBASE_DEBUG_ASSERT(buffer);
1089         KBASE_DEBUG_ASSERT(f_pos);
1090         CSTD_UNUSED(filp);
1091
1092         if ((0 > *f_pos) || (PACKET_SIZE > size))
1093                 return -EINVAL;
1094
1095         mutex_lock(&tl_reader_lock);
1096
1097         while (copy_len < size) {
1098                 enum tl_stream_type stype;
1099                 unsigned int        rb_idx_raw = 0;
1100                 unsigned int        rb_idx;
1101                 size_t              rb_size;
1102
1103                 /* If we don't have any data yet, wait for packet to be
1104                  * submitted. If we already read some packets and there is no
1105                  * packet pending return back to user. */
1106                 if (0 < copy_len) {
1107                         if (!kbasep_tlstream_packet_pending(
1108                                                 &stype,
1109                                                 &rb_idx_raw))
1110                                 break;
1111                 } else {
1112                         if (wait_event_interruptible(
1113                                                 tl_event_queue,
1114                                                 kbasep_tlstream_packet_pending(
1115                                                         &stype,
1116                                                         &rb_idx_raw))) {
1117                                 copy_len = -ERESTARTSYS;
1118                                 break;
1119                         }
1120                 }
1121
1122                 /* Check if this packet fits into the user buffer.
1123                  * If so copy its content. */
1124                 rb_idx = rb_idx_raw % PACKET_COUNT;
1125                 rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size);
1126                 if (rb_size > size - copy_len)
1127                         break;
1128                 if (copy_to_user(
1129                                         &buffer[copy_len],
1130                                         tl_stream[stype]->buffer[rb_idx].data,
1131                                         rb_size)) {
1132                         copy_len = -EFAULT;
1133                         break;
1134                 }
1135
1136                 /* If the rbi still points to the packet we just processed
1137                  * then there was no overflow so we add the copied size to
1138                  * copy_len and move rbi on to the next packet
1139                  */
1140                 smp_rmb();
1141                 if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) {
1142                         copy_len += rb_size;
1143                         atomic_inc(&tl_stream[stype]->rbi);
1144
1145 #if MALI_UNIT_TEST
1146                         atomic_add(rb_size, &tlstream_bytes_collected);
1147 #endif /* MALI_UNIT_TEST */
1148                 }
1149         }
1150
1151         mutex_unlock(&tl_reader_lock);
1152
1153         return copy_len;
1154 }
1155
1156 /**
1157  * kbasep_tlstream_poll - poll timeline stream for packets
1158  * @filp: pointer to file structure
1159  * @wait: pointer to poll table
1160  * Return: POLLIN if data can be read without blocking, otherwise zero
1161  */
1162 static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait)
1163 {
1164         enum tl_stream_type stream_type;
1165         unsigned int        rb_idx;
1166
1167         KBASE_DEBUG_ASSERT(filp);
1168         KBASE_DEBUG_ASSERT(wait);
1169
1170         poll_wait(filp, &tl_event_queue, wait);
1171         if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx))
1172                 return POLLIN;
1173         return 0;
1174 }
1175
1176 /**
1177  * kbasep_tlstream_release - release timeline stream descriptor
1178  * @inode: pointer to inode structure
1179  * @filp:  pointer to file structure
1180  *
1181  * Return always return zero
1182  */
1183 static int kbasep_tlstream_release(struct inode *inode, struct file *filp)
1184 {
1185         KBASE_DEBUG_ASSERT(inode);
1186         KBASE_DEBUG_ASSERT(filp);
1187         CSTD_UNUSED(inode);
1188         CSTD_UNUSED(filp);
1189
1190         /* Stop autoflush timer before releasing access to streams. */
1191         atomic_set(&autoflush_timer_active, 0);
1192         del_timer_sync(&autoflush_timer);
1193
1194         atomic_set(&kbase_tlstream_enabled, 0);
1195         return 0;
1196 }
1197
1198 /**
1199  * kbasep_tlstream_timeline_header - prepare timeline header stream packet
1200  * @stream_type: type of the stream that will carry header data
1201  * @tp_desc:     pointer to array with tracepoint descriptors
1202  * @tp_count:    number of descriptors in the given array
1203  *
1204  * Functions fills in information about tracepoints stored in body stream
1205  * associated with this header stream.
1206  */
1207 static void kbasep_tlstream_timeline_header(
1208                 enum tl_stream_type  stream_type,
1209                 const struct tp_desc *tp_desc,
1210                 u32                  tp_count)
1211 {
1212         const u8      tv = SWTRACE_VERSION; /* protocol version */
1213         const u8      ps = sizeof(void *); /* pointer size */
1214         size_t        msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count);
1215         char          *buffer;
1216         size_t        pos = 0;
1217         unsigned long flags;
1218         unsigned int  i;
1219
1220         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
1221         KBASE_DEBUG_ASSERT(tp_desc);
1222
1223         /* Calculate the size of the timeline message. */
1224         for (i = 0; i < tp_count; i++) {
1225                 msg_size += sizeof(tp_desc[i].id);
1226                 msg_size +=
1227                         strnlen(tp_desc[i].id_str,    STRLEN_MAX) +
1228                         sizeof(char) + sizeof(u32);
1229                 msg_size +=
1230                         strnlen(tp_desc[i].name,      STRLEN_MAX) +
1231                         sizeof(char) + sizeof(u32);
1232                 msg_size +=
1233                         strnlen(tp_desc[i].arg_types, STRLEN_MAX) +
1234                         sizeof(char) + sizeof(u32);
1235                 msg_size +=
1236                         strnlen(tp_desc[i].arg_names, STRLEN_MAX) +
1237                         sizeof(char) + sizeof(u32);
1238         }
1239
1240         KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size);
1241
1242         buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags);
1243         KBASE_DEBUG_ASSERT(buffer);
1244
1245         pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv));
1246         pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps));
1247         pos = kbasep_tlstream_write_bytes(
1248                         buffer, pos, &tp_count, sizeof(tp_count));
1249
1250         for (i = 0; i < tp_count; i++) {
1251                 pos = kbasep_tlstream_write_bytes(
1252                                 buffer, pos,
1253                                 &tp_desc[i].id, sizeof(tp_desc[i].id));
1254                 pos = kbasep_tlstream_write_string(
1255                                 buffer, pos,
1256                                 tp_desc[i].id_str, msg_size - pos);
1257                 pos = kbasep_tlstream_write_string(
1258                                 buffer, pos,
1259                                 tp_desc[i].name, msg_size - pos);
1260                 pos = kbasep_tlstream_write_string(
1261                                 buffer, pos,
1262                                 tp_desc[i].arg_types, msg_size - pos);
1263                 pos = kbasep_tlstream_write_string(
1264                                 buffer, pos,
1265                                 tp_desc[i].arg_names, msg_size - pos);
1266         }
1267
1268         KBASE_DEBUG_ASSERT(msg_size == pos);
1269
1270         kbasep_tlstream_msgbuf_release(stream_type, flags);
1271
1272         /* We don't expect any more data to be read in this stream.
1273          * As header stream must be read before its associated body stream,
1274          * make this packet visible to the user straightaway. */
1275         kbasep_tlstream_flush_stream(stream_type);
1276 }
1277
1278 /*****************************************************************************/
1279
1280 int kbase_tlstream_init(void)
1281 {
1282         enum tl_stream_type i;
1283
1284         /* Prepare stream structures. */
1285         for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
1286                 tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL);
1287                 if (!tl_stream[i])
1288                         break;
1289                 kbasep_timeline_stream_init(tl_stream[i], i);
1290         }
1291         if (TL_STREAM_TYPE_COUNT > i) {
1292                 for (; i > 0; i--) {
1293                         kbasep_timeline_stream_term(tl_stream[i - 1]);
1294                         kfree(tl_stream[i - 1]);
1295                 }
1296                 return -ENOMEM;
1297         }
1298
1299         /* Initialize autoflush timer. */
1300         atomic_set(&autoflush_timer_active, 0);
1301         setup_timer(&autoflush_timer,
1302                         kbasep_tlstream_autoflush_timer_callback,
1303                         0);
1304
1305         return 0;
1306 }
1307
1308 void kbase_tlstream_term(void)
1309 {
1310         enum tl_stream_type i;
1311
1312         for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
1313                 kbasep_timeline_stream_term(tl_stream[i]);
1314                 kfree(tl_stream[i]);
1315         }
1316 }
1317
1318 int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd)
1319 {
1320         if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, 1)) {
1321                 int rcode;
1322
1323                 *fd = anon_inode_getfd(
1324                                 "[mali_tlstream]",
1325                                 &kbasep_tlstream_fops,
1326                                 kctx,
1327                                 O_RDONLY | O_CLOEXEC);
1328                 if (0 > *fd) {
1329                         atomic_set(&kbase_tlstream_enabled, 0);
1330                         return *fd;
1331                 }
1332
1333                 /* Reset and initialize header streams. */
1334                 kbasep_timeline_stream_reset(
1335                                 tl_stream[TL_STREAM_TYPE_OBJ_HEADER]);
1336                 kbasep_timeline_stream_reset(
1337                                 tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]);
1338                 kbasep_timeline_stream_reset(
1339                                 tl_stream[TL_STREAM_TYPE_AUX_HEADER]);
1340                 kbasep_tlstream_timeline_header(
1341                                 TL_STREAM_TYPE_OBJ_HEADER,
1342                                 tp_desc_obj,
1343                                 ARRAY_SIZE(tp_desc_obj));
1344                 kbasep_tlstream_timeline_header(
1345                                 TL_STREAM_TYPE_AUX_HEADER,
1346                                 tp_desc_aux,
1347                                 ARRAY_SIZE(tp_desc_aux));
1348
1349                 /* Start autoflush timer. */
1350                 atomic_set(&autoflush_timer_active, 1);
1351                 rcode = mod_timer(
1352                                 &autoflush_timer,
1353                                 jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
1354                 CSTD_UNUSED(rcode);
1355
1356         } else {
1357                 *fd = -EBUSY;
1358         }
1359
1360         return 0;
1361 }
1362
1363 void kbase_tlstream_flush_streams(void)
1364 {
1365         enum tl_stream_type stype;
1366
1367         for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++)
1368                 kbasep_tlstream_flush_stream(stype);
1369 }
1370
1371 void kbase_tlstream_reset_body_streams(void)
1372 {
1373         kbasep_timeline_stream_reset(
1374                         tl_stream[TL_STREAM_TYPE_OBJ]);
1375         kbasep_timeline_stream_reset(
1376                         tl_stream[TL_STREAM_TYPE_AUX]);
1377 }
1378
1379 #if MALI_UNIT_TEST
1380 void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated)
1381 {
1382         KBASE_DEBUG_ASSERT(bytes_collected);
1383         KBASE_DEBUG_ASSERT(bytes_generated);
1384         *bytes_collected = atomic_read(&tlstream_bytes_collected);
1385         *bytes_generated = atomic_read(&tlstream_bytes_generated);
1386 }
1387 #endif /* MALI_UNIT_TEST */
1388
1389 /*****************************************************************************/
1390
1391 void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid)
1392 {
1393         const u32     msg_id = KBASE_TL_NEW_CTX;
1394         const size_t  msg_size =
1395                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) +
1396                 sizeof(tgid);
1397         unsigned long flags;
1398         char          *buffer;
1399         size_t        pos = 0;
1400
1401         buffer = kbasep_tlstream_msgbuf_acquire(
1402                         TL_STREAM_TYPE_OBJ_SUMMARY,
1403                         msg_size, &flags);
1404         KBASE_DEBUG_ASSERT(buffer);
1405
1406         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1407         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1408         pos = kbasep_tlstream_write_bytes(
1409                         buffer, pos, &context, sizeof(context));
1410         pos = kbasep_tlstream_write_bytes(
1411                         buffer, pos, &nr, sizeof(nr));
1412         pos = kbasep_tlstream_write_bytes(
1413                         buffer, pos, &tgid, sizeof(tgid));
1414
1415         KBASE_DEBUG_ASSERT(msg_size == pos);
1416
1417         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1418 }
1419
1420 void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count)
1421 {
1422         const u32     msg_id = KBASE_TL_NEW_GPU;
1423         const size_t  msg_size =
1424                 sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) +
1425                 sizeof(core_count);
1426         unsigned long flags;
1427         char          *buffer;
1428         size_t        pos = 0;
1429
1430         buffer = kbasep_tlstream_msgbuf_acquire(
1431                         TL_STREAM_TYPE_OBJ_SUMMARY,
1432                         msg_size, &flags);
1433         KBASE_DEBUG_ASSERT(buffer);
1434
1435         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1436         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1437         pos = kbasep_tlstream_write_bytes(
1438                         buffer, pos, &gpu, sizeof(gpu));
1439         pos = kbasep_tlstream_write_bytes(
1440                         buffer, pos, &id, sizeof(id));
1441         pos = kbasep_tlstream_write_bytes(
1442                         buffer, pos, &core_count, sizeof(core_count));
1443         KBASE_DEBUG_ASSERT(msg_size == pos);
1444
1445         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1446 }
1447
1448 void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn)
1449 {
1450         const u32     msg_id = KBASE_TL_NEW_LPU;
1451         const size_t  msg_size =
1452                 sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) +
1453                 sizeof(fn);
1454         unsigned long flags;
1455         char          *buffer;
1456         size_t        pos = 0;
1457
1458         buffer = kbasep_tlstream_msgbuf_acquire(
1459                         TL_STREAM_TYPE_OBJ_SUMMARY,
1460                         msg_size, &flags);
1461         KBASE_DEBUG_ASSERT(buffer);
1462
1463         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1464         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1465         pos = kbasep_tlstream_write_bytes(
1466                         buffer, pos, &lpu, sizeof(lpu));
1467         pos = kbasep_tlstream_write_bytes(
1468                         buffer, pos, &nr, sizeof(nr));
1469         pos = kbasep_tlstream_write_bytes(
1470                         buffer, pos, &fn, sizeof(fn));
1471         KBASE_DEBUG_ASSERT(msg_size == pos);
1472
1473         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1474 }
1475
1476 void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu)
1477 {
1478         const u32     msg_id = KBASE_TL_LIFELINK_LPU_GPU;
1479         const size_t  msg_size =
1480                 sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu);
1481         unsigned long flags;
1482         char          *buffer;
1483         size_t        pos = 0;
1484
1485         buffer = kbasep_tlstream_msgbuf_acquire(
1486                         TL_STREAM_TYPE_OBJ_SUMMARY,
1487                         msg_size, &flags);
1488         KBASE_DEBUG_ASSERT(buffer);
1489
1490         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1491         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1492         pos = kbasep_tlstream_write_bytes(
1493                         buffer, pos, &lpu, sizeof(lpu));
1494         pos = kbasep_tlstream_write_bytes(
1495                         buffer, pos, &gpu, sizeof(gpu));
1496         KBASE_DEBUG_ASSERT(msg_size == pos);
1497
1498         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1499 }
1500
1501 void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr)
1502 {
1503         const u32     msg_id = KBASE_TL_NEW_AS;
1504         const size_t  msg_size =
1505                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr);
1506         unsigned long flags;
1507         char          *buffer;
1508         size_t        pos = 0;
1509
1510         buffer = kbasep_tlstream_msgbuf_acquire(
1511                         TL_STREAM_TYPE_OBJ_SUMMARY,
1512                         msg_size, &flags);
1513         KBASE_DEBUG_ASSERT(buffer);
1514
1515         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1516         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1517         pos = kbasep_tlstream_write_bytes(
1518                         buffer, pos, &as, sizeof(as));
1519         pos = kbasep_tlstream_write_bytes(
1520                         buffer, pos, &nr, sizeof(nr));
1521         KBASE_DEBUG_ASSERT(msg_size == pos);
1522
1523         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1524 }
1525
1526 void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu)
1527 {
1528         const u32     msg_id = KBASE_TL_LIFELINK_AS_GPU;
1529         const size_t  msg_size =
1530                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu);
1531         unsigned long flags;
1532         char          *buffer;
1533         size_t        pos = 0;
1534
1535         buffer = kbasep_tlstream_msgbuf_acquire(
1536                         TL_STREAM_TYPE_OBJ_SUMMARY,
1537                         msg_size, &flags);
1538         KBASE_DEBUG_ASSERT(buffer);
1539
1540         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1541         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1542         pos = kbasep_tlstream_write_bytes(
1543                         buffer, pos, &as, sizeof(as));
1544         pos = kbasep_tlstream_write_bytes(
1545                         buffer, pos, &gpu, sizeof(gpu));
1546         KBASE_DEBUG_ASSERT(msg_size == pos);
1547
1548         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1549 }
1550
1551 /*****************************************************************************/
1552
1553 void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid)
1554 {
1555         const u32     msg_id = KBASE_TL_NEW_CTX;
1556         const size_t  msg_size =
1557                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) +
1558                 sizeof(tgid);
1559         unsigned long flags;
1560         char          *buffer;
1561         size_t        pos = 0;
1562
1563         buffer = kbasep_tlstream_msgbuf_acquire(
1564                         TL_STREAM_TYPE_OBJ,
1565                         msg_size, &flags);
1566         KBASE_DEBUG_ASSERT(buffer);
1567
1568         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1569         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1570         pos = kbasep_tlstream_write_bytes(
1571                         buffer, pos, &context, sizeof(context));
1572         pos = kbasep_tlstream_write_bytes(
1573                         buffer, pos, &nr, sizeof(nr));
1574         pos = kbasep_tlstream_write_bytes(
1575                         buffer, pos, &tgid, sizeof(tgid));
1576         KBASE_DEBUG_ASSERT(msg_size == pos);
1577
1578         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1579 }
1580
1581 void __kbase_tlstream_tl_new_atom(void *atom, u32 nr)
1582 {
1583         const u32     msg_id = KBASE_TL_NEW_ATOM;
1584         const size_t  msg_size =
1585                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(nr);
1586         unsigned long flags;
1587         char          *buffer;
1588         size_t        pos = 0;
1589
1590         buffer = kbasep_tlstream_msgbuf_acquire(
1591                         TL_STREAM_TYPE_OBJ,
1592                         msg_size, &flags);
1593         KBASE_DEBUG_ASSERT(buffer);
1594
1595         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1596         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1597         pos = kbasep_tlstream_write_bytes(
1598                         buffer, pos, &atom, sizeof(atom));
1599         pos = kbasep_tlstream_write_bytes(
1600                         buffer, pos, &nr, sizeof(nr));
1601         KBASE_DEBUG_ASSERT(msg_size == pos);
1602
1603         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1604 }
1605
1606 void __kbase_tlstream_tl_del_ctx(void *context)
1607 {
1608         const u32     msg_id = KBASE_TL_DEL_CTX;
1609         const size_t  msg_size =
1610                 sizeof(msg_id) + sizeof(u64) + sizeof(context);
1611         unsigned long flags;
1612         char          *buffer;
1613         size_t        pos = 0;
1614
1615         buffer = kbasep_tlstream_msgbuf_acquire(
1616                         TL_STREAM_TYPE_OBJ,
1617                         msg_size, &flags);
1618         KBASE_DEBUG_ASSERT(buffer);
1619
1620         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1621         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1622         pos = kbasep_tlstream_write_bytes(
1623                         buffer, pos, &context, sizeof(context));
1624         KBASE_DEBUG_ASSERT(msg_size == pos);
1625
1626         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1627 }
1628
1629 void __kbase_tlstream_tl_del_atom(void *atom)
1630 {
1631         const u32     msg_id = KBASE_TL_DEL_ATOM;
1632         const size_t  msg_size =
1633                 sizeof(msg_id) + sizeof(u64) + sizeof(atom);
1634         unsigned long flags;
1635         char          *buffer;
1636         size_t        pos = 0;
1637
1638         buffer = kbasep_tlstream_msgbuf_acquire(
1639                         TL_STREAM_TYPE_OBJ,
1640                         msg_size, &flags);
1641         KBASE_DEBUG_ASSERT(buffer);
1642
1643         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1644         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1645         pos = kbasep_tlstream_write_bytes(
1646                         buffer, pos, &atom, sizeof(atom));
1647         KBASE_DEBUG_ASSERT(msg_size == pos);
1648
1649         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1650 }
1651
1652 void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu)
1653 {
1654         const u32     msg_id = KBASE_TL_RET_CTX_LPU;
1655         const size_t  msg_size =
1656                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu);
1657         unsigned long flags;
1658         char          *buffer;
1659         size_t        pos = 0;
1660
1661         buffer = kbasep_tlstream_msgbuf_acquire(
1662                         TL_STREAM_TYPE_OBJ,
1663                         msg_size, &flags);
1664         KBASE_DEBUG_ASSERT(buffer);
1665
1666         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1667         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1668         pos = kbasep_tlstream_write_bytes(
1669                         buffer, pos, &context, sizeof(context));
1670         pos = kbasep_tlstream_write_bytes(
1671                         buffer, pos, &lpu, sizeof(lpu));
1672         KBASE_DEBUG_ASSERT(msg_size == pos);
1673
1674         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1675 }
1676
1677 void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context)
1678 {
1679         const u32     msg_id = KBASE_TL_RET_ATOM_CTX;
1680         const size_t  msg_size =
1681                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
1682         unsigned long flags;
1683         char          *buffer;
1684         size_t        pos = 0;
1685
1686         buffer = kbasep_tlstream_msgbuf_acquire(
1687                         TL_STREAM_TYPE_OBJ,
1688                         msg_size, &flags);
1689         KBASE_DEBUG_ASSERT(buffer);
1690
1691         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1692         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1693         pos = kbasep_tlstream_write_bytes(
1694                         buffer, pos, &atom, sizeof(atom));
1695         pos = kbasep_tlstream_write_bytes(
1696                         buffer, pos, &context, sizeof(context));
1697         KBASE_DEBUG_ASSERT(msg_size == pos);
1698
1699         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1700 }
1701
1702 void __kbase_tlstream_tl_ret_atom_lpu(
1703                 void *atom, void *lpu, const char *attrib_match_list)
1704 {
1705         const u32     msg_id = KBASE_TL_RET_ATOM_LPU;
1706         const size_t  msg_s0 = sizeof(u32) + sizeof(char) +
1707                         strnlen(attrib_match_list, STRLEN_MAX);
1708         const size_t  msg_size =
1709                         sizeof(msg_id) + sizeof(u64) +
1710                         sizeof(atom) + sizeof(lpu) + msg_s0;
1711         unsigned long flags;
1712         char          *buffer;
1713         size_t        pos = 0;
1714
1715         buffer = kbasep_tlstream_msgbuf_acquire(
1716                         TL_STREAM_TYPE_OBJ,
1717                         msg_size, &flags);
1718         KBASE_DEBUG_ASSERT(buffer);
1719
1720         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1721         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1722         pos = kbasep_tlstream_write_bytes(
1723                         buffer, pos, &atom, sizeof(atom));
1724         pos = kbasep_tlstream_write_bytes(
1725                         buffer, pos, &lpu, sizeof(lpu));
1726         pos = kbasep_tlstream_write_string(
1727                         buffer, pos, attrib_match_list, msg_s0);
1728         KBASE_DEBUG_ASSERT(msg_size == pos);
1729
1730         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1731 }
1732
1733 void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu)
1734 {
1735         const u32     msg_id = KBASE_TL_NRET_CTX_LPU;
1736         const size_t  msg_size =
1737                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu);
1738         unsigned long flags;
1739         char          *buffer;
1740         size_t        pos = 0;
1741
1742         buffer = kbasep_tlstream_msgbuf_acquire(
1743                         TL_STREAM_TYPE_OBJ,
1744                         msg_size, &flags);
1745         KBASE_DEBUG_ASSERT(buffer);
1746
1747         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1748         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1749         pos = kbasep_tlstream_write_bytes(
1750                         buffer, pos, &context, sizeof(context));
1751         pos = kbasep_tlstream_write_bytes(
1752                         buffer, pos, &lpu, sizeof(lpu));
1753         KBASE_DEBUG_ASSERT(msg_size == pos);
1754
1755         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1756 }
1757
1758 void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context)
1759 {
1760         const u32     msg_id = KBASE_TL_NRET_ATOM_CTX;
1761         const size_t  msg_size =
1762                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
1763         unsigned long flags;
1764         char          *buffer;
1765         size_t        pos = 0;
1766
1767         buffer = kbasep_tlstream_msgbuf_acquire(
1768                         TL_STREAM_TYPE_OBJ,
1769                         msg_size, &flags);
1770         KBASE_DEBUG_ASSERT(buffer);
1771
1772         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1773         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1774         pos = kbasep_tlstream_write_bytes(
1775                         buffer, pos, &atom, sizeof(atom));
1776         pos = kbasep_tlstream_write_bytes(
1777                         buffer, pos, &context, sizeof(context));
1778         KBASE_DEBUG_ASSERT(msg_size == pos);
1779
1780         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1781 }
1782
1783 void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2)
1784 {
1785         const u32     msg_id = KBASE_TL_DEP_ATOM_ATOM;
1786         const size_t  msg_size =
1787                 sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
1788         unsigned long flags;
1789         char          *buffer;
1790         size_t        pos = 0;
1791
1792         buffer = kbasep_tlstream_msgbuf_acquire(
1793                         TL_STREAM_TYPE_OBJ,
1794                         msg_size, &flags);
1795         KBASE_DEBUG_ASSERT(buffer);
1796
1797         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1798         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1799         pos = kbasep_tlstream_write_bytes(
1800                         buffer, pos, &atom1, sizeof(atom1));
1801         pos = kbasep_tlstream_write_bytes(
1802                         buffer, pos, &atom2, sizeof(atom2));
1803         KBASE_DEBUG_ASSERT(msg_size == pos);
1804
1805         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1806 }
1807
1808 void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2)
1809 {
1810         const u32     msg_id = KBASE_TL_NDEP_ATOM_ATOM;
1811         const size_t  msg_size =
1812                 sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
1813         unsigned long flags;
1814         char          *buffer;
1815         size_t        pos = 0;
1816
1817         buffer = kbasep_tlstream_msgbuf_acquire(
1818                         TL_STREAM_TYPE_OBJ,
1819                         msg_size, &flags);
1820         KBASE_DEBUG_ASSERT(buffer);
1821
1822         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1823         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1824         pos = kbasep_tlstream_write_bytes(
1825                         buffer, pos, &atom1, sizeof(atom1));
1826         pos = kbasep_tlstream_write_bytes(
1827                         buffer, pos, &atom2, sizeof(atom2));
1828         KBASE_DEBUG_ASSERT(msg_size == pos);
1829
1830         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1831 }
1832
1833 void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2)
1834 {
1835         const u32     msg_id = KBASE_TL_RDEP_ATOM_ATOM;
1836         const size_t  msg_size =
1837                 sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
1838         unsigned long flags;
1839         char          *buffer;
1840         size_t        pos = 0;
1841
1842         buffer = kbasep_tlstream_msgbuf_acquire(
1843                         TL_STREAM_TYPE_OBJ,
1844                         msg_size, &flags);
1845         KBASE_DEBUG_ASSERT(buffer);
1846
1847         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1848         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1849         pos = kbasep_tlstream_write_bytes(
1850                         buffer, pos, &atom1, sizeof(atom1));
1851         pos = kbasep_tlstream_write_bytes(
1852                         buffer, pos, &atom2, sizeof(atom2));
1853         KBASE_DEBUG_ASSERT(msg_size == pos);
1854
1855         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1856 }
1857
1858 void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu)
1859 {
1860         const u32     msg_id = KBASE_TL_NRET_ATOM_LPU;
1861         const size_t  msg_size =
1862                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
1863         unsigned long flags;
1864         char          *buffer;
1865         size_t        pos = 0;
1866
1867         buffer = kbasep_tlstream_msgbuf_acquire(
1868                         TL_STREAM_TYPE_OBJ,
1869                         msg_size, &flags);
1870         KBASE_DEBUG_ASSERT(buffer);
1871
1872         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1873         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1874         pos = kbasep_tlstream_write_bytes(
1875                         buffer, pos, &atom, sizeof(atom));
1876         pos = kbasep_tlstream_write_bytes(
1877                         buffer, pos, &lpu, sizeof(lpu));
1878         KBASE_DEBUG_ASSERT(msg_size == pos);
1879
1880         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1881 }
1882
1883 void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx)
1884 {
1885         const u32     msg_id = KBASE_TL_RET_AS_CTX;
1886         const size_t  msg_size =
1887                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx);
1888         unsigned long flags;
1889         char          *buffer;
1890         size_t        pos = 0;
1891
1892         buffer = kbasep_tlstream_msgbuf_acquire(
1893                         TL_STREAM_TYPE_OBJ,
1894                         msg_size, &flags);
1895         KBASE_DEBUG_ASSERT(buffer);
1896
1897         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1898         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1899         pos = kbasep_tlstream_write_bytes(
1900                         buffer, pos, &as, sizeof(as));
1901         pos = kbasep_tlstream_write_bytes(
1902                         buffer, pos, &ctx, sizeof(ctx));
1903         KBASE_DEBUG_ASSERT(msg_size == pos);
1904
1905         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1906 }
1907
1908 void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx)
1909 {
1910         const u32     msg_id = KBASE_TL_NRET_AS_CTX;
1911         const size_t  msg_size =
1912                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx);
1913         unsigned long flags;
1914         char          *buffer;
1915         size_t        pos = 0;
1916
1917         buffer = kbasep_tlstream_msgbuf_acquire(
1918                         TL_STREAM_TYPE_OBJ,
1919                         msg_size, &flags);
1920         KBASE_DEBUG_ASSERT(buffer);
1921
1922         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1923         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1924         pos = kbasep_tlstream_write_bytes(
1925                         buffer, pos, &as, sizeof(as));
1926         pos = kbasep_tlstream_write_bytes(
1927                         buffer, pos, &ctx, sizeof(ctx));
1928         KBASE_DEBUG_ASSERT(msg_size == pos);
1929
1930         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1931 }
1932
1933 void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as)
1934 {
1935         const u32     msg_id = KBASE_TL_RET_ATOM_AS;
1936         const size_t  msg_size =
1937                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as);
1938         unsigned long flags;
1939         char          *buffer;
1940         size_t        pos = 0;
1941
1942         buffer = kbasep_tlstream_msgbuf_acquire(
1943                         TL_STREAM_TYPE_OBJ,
1944                         msg_size, &flags);
1945         KBASE_DEBUG_ASSERT(buffer);
1946
1947         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1948         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1949         pos = kbasep_tlstream_write_bytes(
1950                         buffer, pos, &atom, sizeof(atom));
1951         pos = kbasep_tlstream_write_bytes(
1952                         buffer, pos, &as, sizeof(as));
1953         KBASE_DEBUG_ASSERT(msg_size == pos);
1954
1955         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1956 }
1957
1958 void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as)
1959 {
1960         const u32     msg_id = KBASE_TL_NRET_ATOM_AS;
1961         const size_t  msg_size =
1962                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as);
1963         unsigned long flags;
1964         char          *buffer;
1965         size_t        pos = 0;
1966
1967         buffer = kbasep_tlstream_msgbuf_acquire(
1968                         TL_STREAM_TYPE_OBJ,
1969                         msg_size, &flags);
1970         KBASE_DEBUG_ASSERT(buffer);
1971
1972         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1973         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1974         pos = kbasep_tlstream_write_bytes(
1975                         buffer, pos, &atom, sizeof(atom));
1976         pos = kbasep_tlstream_write_bytes(
1977                         buffer, pos, &as, sizeof(as));
1978         KBASE_DEBUG_ASSERT(msg_size == pos);
1979
1980         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1981 }
1982
1983 void __kbase_tlstream_tl_attrib_atom_config(
1984                 void *atom, u64 jd, u64 affinity, u32 config)
1985 {
1986         const u32     msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG;
1987         const size_t  msg_size =
1988                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) +
1989                 sizeof(jd) + sizeof(affinity) + sizeof(config);
1990         unsigned long flags;
1991         char          *buffer;
1992         size_t        pos = 0;
1993
1994         buffer = kbasep_tlstream_msgbuf_acquire(
1995                         TL_STREAM_TYPE_OBJ,
1996                         msg_size, &flags);
1997         KBASE_DEBUG_ASSERT(buffer);
1998
1999         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2000         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2001         pos = kbasep_tlstream_write_bytes(
2002                         buffer, pos, &atom, sizeof(atom));
2003         pos = kbasep_tlstream_write_bytes(
2004                         buffer, pos, &jd, sizeof(jd));
2005         pos = kbasep_tlstream_write_bytes(
2006                         buffer, pos, &affinity, sizeof(affinity));
2007         pos = kbasep_tlstream_write_bytes(
2008                         buffer, pos, &config, sizeof(config));
2009         KBASE_DEBUG_ASSERT(msg_size == pos);
2010
2011         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
2012 }
2013
2014 void __kbase_tlstream_tl_attrib_as_config(
2015                 void *as, u64 transtab, u64 memattr, u64 transcfg)
2016 {
2017         const u32     msg_id = KBASE_TL_ATTRIB_AS_CONFIG;
2018         const size_t  msg_size =
2019                 sizeof(msg_id) + sizeof(u64) + sizeof(as) +
2020                 sizeof(transtab) + sizeof(memattr) + sizeof(transcfg);
2021         unsigned long flags;
2022         char          *buffer;
2023         size_t        pos = 0;
2024
2025         buffer = kbasep_tlstream_msgbuf_acquire(
2026                         TL_STREAM_TYPE_OBJ,
2027                         msg_size, &flags);
2028         KBASE_DEBUG_ASSERT(buffer);
2029
2030         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2031         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2032         pos = kbasep_tlstream_write_bytes(
2033                         buffer, pos, &as, sizeof(as));
2034         pos = kbasep_tlstream_write_bytes(
2035                         buffer, pos, &transtab, sizeof(transtab));
2036         pos = kbasep_tlstream_write_bytes(
2037                         buffer, pos, &memattr, sizeof(memattr));
2038         pos = kbasep_tlstream_write_bytes(
2039                         buffer, pos, &transcfg, sizeof(transcfg));
2040         KBASE_DEBUG_ASSERT(msg_size == pos);
2041
2042         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
2043 }
2044
2045 void __kbase_tlstream_jd_gpu_soft_reset(void *gpu)
2046 {
2047         const u32     msg_id = KBASE_JD_GPU_SOFT_RESET;
2048         const size_t  msg_size =
2049                 sizeof(msg_id) + sizeof(u64) + sizeof(gpu);
2050         unsigned long flags;
2051         char          *buffer;
2052         size_t        pos = 0;
2053
2054         buffer = kbasep_tlstream_msgbuf_acquire(
2055                         TL_STREAM_TYPE_OBJ,
2056                         msg_size, &flags);
2057         KBASE_DEBUG_ASSERT(buffer);
2058
2059         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2060         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2061         pos = kbasep_tlstream_write_bytes(
2062                         buffer, pos, &gpu, sizeof(gpu));
2063         KBASE_DEBUG_ASSERT(msg_size == pos);
2064
2065         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
2066 }
2067
2068 /*****************************************************************************/
2069
2070 void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state)
2071 {
2072         const u32     msg_id = KBASE_AUX_PM_STATE;
2073         const size_t  msg_size =
2074                 sizeof(msg_id) + sizeof(u64) + sizeof(core_type) +
2075                 sizeof(state);
2076         unsigned long flags;
2077         char          *buffer;
2078         size_t        pos = 0;
2079
2080         buffer = kbasep_tlstream_msgbuf_acquire(
2081                         TL_STREAM_TYPE_AUX,
2082                         msg_size, &flags);
2083         KBASE_DEBUG_ASSERT(buffer);
2084
2085         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2086         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2087         pos = kbasep_tlstream_write_bytes(
2088                         buffer, pos, &core_type, sizeof(core_type));
2089         pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state));
2090         KBASE_DEBUG_ASSERT(msg_size == pos);
2091
2092         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2093 }
2094
2095 void __kbase_tlstream_aux_issue_job_softstop(void *katom)
2096 {
2097         const u32     msg_id = KBASE_AUX_ISSUE_JOB_SOFTSTOP;
2098         const size_t  msg_size =
2099                 sizeof(msg_id) + sizeof(u64) + sizeof(katom);
2100         unsigned long flags;
2101         char          *buffer;
2102         size_t        pos = 0;
2103
2104         buffer = kbasep_tlstream_msgbuf_acquire(
2105                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2106         KBASE_DEBUG_ASSERT(buffer);
2107
2108         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2109         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2110         pos = kbasep_tlstream_write_bytes(buffer, pos, &katom, sizeof(katom));
2111         KBASE_DEBUG_ASSERT(msg_size == pos);
2112
2113         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2114 }
2115
2116 void __kbase_tlstream_aux_job_softstop(u32 js_id)
2117 {
2118         const u32     msg_id = KBASE_AUX_JOB_SOFTSTOP;
2119         const size_t  msg_size =
2120                 sizeof(msg_id) + sizeof(u64) + sizeof(js_id);
2121         unsigned long flags;
2122         char          *buffer;
2123         size_t        pos = 0;
2124
2125         buffer = kbasep_tlstream_msgbuf_acquire(
2126                         TL_STREAM_TYPE_AUX,
2127                         msg_size, &flags);
2128         KBASE_DEBUG_ASSERT(buffer);
2129
2130         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2131         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2132         pos = kbasep_tlstream_write_bytes(buffer, pos, &js_id, sizeof(js_id));
2133         KBASE_DEBUG_ASSERT(msg_size == pos);
2134
2135         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2136 }
2137
2138 /**
2139  * __kbase_tlstream_aux_job_softstop_ex_record - record the trace point
2140  * @katom: the atom that has been soft-stopped
2141  * @job_type: the job type
2142  */
2143 static void __kbase_tlstream_aux_job_softstop_ex_record(
2144                 void *katom, u32 job_type)
2145 {
2146         const u32     msg_id = KBASE_AUX_JOB_SOFTSTOP_EX;
2147         const size_t  msg_size =
2148                 sizeof(msg_id) + sizeof(u64) + sizeof(katom) + sizeof(job_type);
2149         unsigned long flags;
2150         char          *buffer;
2151         size_t        pos = 0;
2152
2153         buffer = kbasep_tlstream_msgbuf_acquire(
2154                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2155         KBASE_DEBUG_ASSERT(buffer);
2156
2157         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2158         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2159         pos = kbasep_tlstream_write_bytes(buffer, pos, &katom, sizeof(katom));
2160         pos = kbasep_tlstream_write_bytes(
2161                         buffer, pos, &job_type, sizeof(job_type));
2162         KBASE_DEBUG_ASSERT(msg_size == pos);
2163
2164         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2165 }
2166
2167 void __kbase_tlstream_aux_job_softstop_ex(struct kbase_jd_atom *katom)
2168 {
2169         struct kbase_context *kctx = katom->kctx;
2170         u64 jd = katom->jc;
2171
2172         while (jd != 0) {
2173                 struct job_descriptor_header *job;
2174                 struct kbase_vmap_struct map;
2175
2176                 job = kbase_vmap(kctx, jd, sizeof(*job), &map);
2177                 if (!job) {
2178                         dev_err(kctx->kbdev->dev,
2179                                 "__kbase_tlstream_aux_job_softstop_ex: failed to map job descriptor 0x%llx for atom 0x%p\n",
2180                                 jd, (void *)katom);
2181                         break;
2182                 }
2183                 if (job->exception_status != BASE_JD_EVENT_STOPPED) {
2184                         kbase_vunmap(kctx, &map);
2185                         break;
2186                 }
2187
2188                 __kbase_tlstream_aux_job_softstop_ex_record(
2189                                 katom, job->job_type);
2190
2191                 jd = job->job_descriptor_size ?
2192                         job->next_job._64 : job->next_job._32;
2193                 kbase_vunmap(kctx, &map);
2194         }
2195 }
2196
2197 void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change)
2198 {
2199         const u32     msg_id = KBASE_AUX_PAGEFAULT;
2200         const size_t  msg_size =
2201                 sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) +
2202                 sizeof(page_count_change);
2203         unsigned long flags;
2204         char          *buffer;
2205         size_t        pos = 0;
2206
2207         buffer = kbasep_tlstream_msgbuf_acquire(
2208                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2209         KBASE_DEBUG_ASSERT(buffer);
2210
2211         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2212         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2213         pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr));
2214         pos = kbasep_tlstream_write_bytes(
2215                         buffer, pos,
2216                         &page_count_change, sizeof(page_count_change));
2217         KBASE_DEBUG_ASSERT(msg_size == pos);
2218
2219         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2220 }
2221
2222 void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count)
2223 {
2224         const u32     msg_id = KBASE_AUX_PAGESALLOC;
2225         const size_t  msg_size =
2226                 sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) +
2227                 sizeof(page_count);
2228         unsigned long flags;
2229         char          *buffer;
2230         size_t        pos = 0;
2231
2232         buffer = kbasep_tlstream_msgbuf_acquire(
2233                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2234         KBASE_DEBUG_ASSERT(buffer);
2235
2236         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2237         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2238         pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr));
2239         pos = kbasep_tlstream_write_bytes(
2240                         buffer, pos, &page_count, sizeof(page_count));
2241         KBASE_DEBUG_ASSERT(msg_size == pos);
2242
2243         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2244 }
2245