drivers/gpu/arm/midgard/mali_kbase_tlstream.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <linux/anon_inodes.h>
  19 #include <linux/atomic.h>
  20 #include <linux/file.h>
  21 #include <linux/mutex.h>
  22 #include <linux/poll.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/string.h>
  25 #include <linux/stringify.h>
  26 #include <linux/timer.h>
  27 #include <linux/wait.h>
  28
  29 #include <mali_kbase.h>
  30 #include <mali_kbase_jm.h>
  31 #include <mali_kbase_tlstream.h>
  32 #include <backend/gpu/mali_kbase_device_internal.h>
  33
  34 /*****************************************************************************/
  35
  36 /* The version of swtrace protocol used in timeline stream. */
  37 #define SWTRACE_VERSION    3
  38
  39 /* The maximum expected length of string in tracepoint descriptor. */
  40 #define STRLEN_MAX         64 /* bytes */
  41
  42 /* The number of nanoseconds in a second. */
  43 #define NSECS_IN_SEC       1000000000ull /* ns */
  44
  45 /* The period of autoflush checker execution in milliseconds. */
  46 #define AUTOFLUSH_INTERVAL 1000 /* ms */
  47
  48 /* The maximum size of a single packet used by timeline. */
  49 #define PACKET_SIZE        4096 /* bytes */
  50
  51 /* The number of packets used by one timeline stream. */
  52 #define PACKET_COUNT       16
  53
  54 /* The number of bytes reserved for packet header.
  55  * These value must be defined according to MIPE documentation. */
  56 #define PACKET_HEADER_SIZE 8 /* bytes */
  57
  58 /* The number of bytes reserved for packet sequence number.
  59  * These value must be defined according to MIPE documentation. */
  60 #define PACKET_NUMBER_SIZE 4 /* bytes */
  61
  62 /* Packet header - first word.
  63  * These values must be defined according to MIPE documentation. */
  64 #define PACKET_STREAMID_POS  0
  65 #define PACKET_STREAMID_LEN  8
  66 #define PACKET_RSVD1_POS     (PACKET_STREAMID_POS + PACKET_STREAMID_LEN)
  67 #define PACKET_RSVD1_LEN     8
  68 #define PACKET_TYPE_POS      (PACKET_RSVD1_POS + PACKET_RSVD1_LEN)
  69 #define PACKET_TYPE_LEN      3
  70 #define PACKET_CLASS_POS     (PACKET_TYPE_POS + PACKET_TYPE_LEN)
  71 #define PACKET_CLASS_LEN     7
  72 #define PACKET_FAMILY_POS    (PACKET_CLASS_POS + PACKET_CLASS_LEN)
  73 #define PACKET_FAMILY_LEN    6
  74
  75 /* Packet header - second word
  76  * These values must be defined according to MIPE documentation. */
  77 #define PACKET_LENGTH_POS    0
  78 #define PACKET_LENGTH_LEN    24
  79 #define PACKET_SEQBIT_POS    (PACKET_LENGTH_POS + PACKET_LENGTH_LEN)
  80 #define PACKET_SEQBIT_LEN    1
  81 #define PACKET_RSVD2_POS     (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN)
  82 #define PACKET_RSVD2_LEN     7
  83
  84 /* Types of streams generated by timeline.
  85  * Order is significant! Header streams must precede respective body streams. */
  86 enum tl_stream_type {
  87         TL_STREAM_TYPE_OBJ_HEADER,
  88         TL_STREAM_TYPE_OBJ_SUMMARY,
  89         TL_STREAM_TYPE_OBJ,
  90         TL_STREAM_TYPE_AUX_HEADER,
  91         TL_STREAM_TYPE_AUX,
  92
  93         TL_STREAM_TYPE_COUNT
  94 };
  95
  96 /* Timeline packet family ids.
  97  * Values are significant! Check MIPE documentation. */
  98 enum tl_packet_family {
  99         TL_PACKET_FAMILY_CTRL = 0, /* control packets */
 100         TL_PACKET_FAMILY_TL   = 1, /* timeline packets */
 101
 102         TL_PACKET_FAMILY_COUNT
 103 };
 104
 105 /* Packet classes used in timeline streams.
 106  * Values are significant! Check MIPE documentation. */
 107 enum tl_packet_class {
 108         TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */
 109         TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */
 110 };
 111
 112 /* Packet types used in timeline streams.
 113  * Values are significant! Check MIPE documentation. */
 114 enum tl_packet_type {
 115         TL_PACKET_TYPE_HEADER  = 0, /* stream's header/directory */
 116         TL_PACKET_TYPE_BODY    = 1, /* stream's body */
 117         TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */
 118 };
 119
 120 /* Message ids of trace events that are recorded in the timeline stream. */
 121 enum tl_msg_id_obj {
 122         /* Timeline object events. */
 123         KBASE_TL_NEW_CTX,
 124         KBASE_TL_NEW_GPU,
 125         KBASE_TL_NEW_LPU,
 126         KBASE_TL_NEW_ATOM,
 127         KBASE_TL_NEW_AS,
 128         KBASE_TL_DEL_CTX,
 129         KBASE_TL_DEL_ATOM,
 130         KBASE_TL_LIFELINK_LPU_GPU,
 131         KBASE_TL_LIFELINK_AS_GPU,
 132         KBASE_TL_RET_CTX_LPU,
 133         KBASE_TL_RET_ATOM_CTX,
 134         KBASE_TL_RET_ATOM_LPU,
 135         KBASE_TL_NRET_CTX_LPU,
 136         KBASE_TL_NRET_ATOM_CTX,
 137         KBASE_TL_NRET_ATOM_LPU,
 138         KBASE_TL_RET_AS_CTX,
 139         KBASE_TL_NRET_AS_CTX,
 140         KBASE_TL_RET_ATOM_AS,
 141         KBASE_TL_NRET_ATOM_AS,
 142         KBASE_TL_DEP_ATOM_ATOM,
 143         KBASE_TL_ATTRIB_ATOM_CONFIG,
 144         KBASE_TL_ATTRIB_AS_CONFIG,
 145
 146         /* Job dump specific events. */
 147         KBASE_JD_GPU_SOFT_RESET
 148 };
 149
 150 /* Message ids of trace events that are recorded in the auxiliary stream. */
 151 enum tl_msg_id_aux {
 152         KBASE_AUX_PM_STATE,
 153         KBASE_AUX_ISSUE_JOB_SOFTSTOP,
 154         KBASE_AUX_JOB_SOFTSTOP,
 155         KBASE_AUX_JOB_SOFTSTOP_EX,
 156         KBASE_AUX_PAGEFAULT,
 157         KBASE_AUX_PAGESALLOC
 158 };
 159
 160 /*****************************************************************************/
 161
 162 /**
 163  * struct tl_stream - timeline stream structure
 164  * @lock: message order lock
 165  * @buffer: array of buffers
 166  * @wbi: write buffer index
 167  * @rbi: read buffer index
 168  * @numbered: if non-zero stream's packets are sequentially numbered
 169  * @autoflush_counter: counter tracking stream's autoflush state
 170  *
 171  * This structure holds information needed to construct proper packets in the
 172  * timeline stream. Each message in sequence must bear timestamp that is greater
 173  * to one in previous message in the same stream. For this reason lock is held
 174  * throughout the process of message creation. Each stream contains set of
 175  * buffers. Each buffer will hold one MIPE packet. In case there is no free
 176  * space required to store incoming message the oldest buffer is discarded.
 177  * Each packet in timeline body stream has sequence number embedded (this value
 178  * must increment monotonically and is used by packets receiver to discover
 179  * buffer overflows.
 180  * Autoflush counter is set to negative number when there is no data pending
 181  * for flush and it is set to zero on every update of the buffer. Autoflush
 182  * timer will increment the counter by one on every expiry. In case there will
 183  * be no activity on the buffer during two consecutive timer expiries, stream
 184  * buffer will be flushed.
 185  */
 186 struct tl_stream {
 187         spinlock_t lock;
 188
 189         struct {
 190                 atomic_t size;              /* number of bytes in buffer */
 191                 char     data[PACKET_SIZE]; /* buffer's data */
 192         } buffer[PACKET_COUNT];
 193
 194         atomic_t wbi;
 195         atomic_t rbi;
 196
 197         int      numbered;
 198         atomic_t autoflush_counter;
 199 };
 200
 201 /**
 202  * struct tp_desc - tracepoint message descriptor structure
 203  * @id:        tracepoint ID identifying message in stream
 204  * @id_str:    human readable version of tracepoint ID
 205  * @name:      tracepoint description
 206  * @arg_types: tracepoint's arguments types declaration
 207  * @arg_names: comma separated list of tracepoint's arguments names
 208  */
 209 struct tp_desc {
 210         u32        id;
 211         const char *id_str;
 212         const char *name;
 213         const char *arg_types;
 214         const char *arg_names;
 215 };
 216
 217 /*****************************************************************************/
 218
 219 /* Configuration of timeline streams generated by kernel.
 220  * Kernel emit only streams containing either timeline object events or
 221  * auxiliary events. All streams have stream id value of 1 (as opposed to user
 222  * space streams that have value of 0). */
 223 static const struct {
 224         enum tl_packet_family pkt_family;
 225         enum tl_packet_class  pkt_class;
 226         enum tl_packet_type   pkt_type;
 227         unsigned int          stream_id;
 228 } tl_stream_cfg[TL_STREAM_TYPE_COUNT] = {
 229         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER,  1},
 230         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1},
 231         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY,    1},
 232         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER,  1},
 233         {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY,    1}
 234 };
 235
 236 /* The timeline streams generated by kernel. */
 237 static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT];
 238
 239 /* Autoflush timer. */
 240 static struct timer_list autoflush_timer;
 241
 242 /* If non-zero autoflush timer is active. */
 243 static atomic_t autoflush_timer_active;
 244
 245 /* Reader lock. Only one reader is allowed to have access to the timeline
 246  * streams at any given time. */
 247 static DEFINE_MUTEX(tl_reader_lock);
 248
 249 /* Timeline stream event queue. */
 250 static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue);
 251
 252 /* The timeline stream file operations functions. */
 253 static ssize_t kbasep_tlstream_read(
 254                 struct file *filp,
 255                 char __user *buffer,
 256                 size_t      size,
 257                 loff_t      *f_pos);
 258 static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait);
 259 static int kbasep_tlstream_release(struct inode *inode, struct file *filp);
 260
 261 /* The timeline stream file operations structure. */
 262 static const struct file_operations kbasep_tlstream_fops = {
 263         .release = kbasep_tlstream_release,
 264         .read    = kbasep_tlstream_read,
 265         .poll    = kbasep_tlstream_poll,
 266 };
 267
 268 /* Descriptors of timeline messages transmitted in object events stream. */
 269 static const struct tp_desc tp_desc_obj[] = {
 270         {
 271                 KBASE_TL_NEW_CTX,
 272                 __stringify(KBASE_TL_NEW_CTX),
 273                 "object ctx is created",
 274                 "@pII",
 275                 "ctx,ctx_nr,tgid"
 276         },
 277         {
 278                 KBASE_TL_NEW_GPU,
 279                 __stringify(KBASE_TL_NEW_GPU),
 280                 "object gpu is created",
 281                 "@pII",
 282                 "gpu,gpu_id,core_count"
 283         },
 284         {
 285                 KBASE_TL_NEW_LPU,
 286                 __stringify(KBASE_TL_NEW_LPU),
 287                 "object lpu is created",
 288                 "@pII",
 289                 "lpu,lpu_nr,lpu_fn"
 290         },
 291         {
 292                 KBASE_TL_NEW_ATOM,
 293                 __stringify(KBASE_TL_NEW_ATOM),
 294                 "object atom is created",
 295                 "@pI",
 296                 "atom,atom_nr"
 297         },
 298         {
 299                 KBASE_TL_NEW_AS,
 300                 __stringify(KBASE_TL_NEW_AS),
 301                 "address space object is created",
 302                 "@pI",
 303                 "address_space,as_nr"
 304         },
 305         {
 306                 KBASE_TL_DEL_CTX,
 307                 __stringify(KBASE_TL_DEL_CTX),
 308                 "context is destroyed",
 309                 "@p",
 310                 "ctx"
 311         },
 312         {
 313                 KBASE_TL_DEL_ATOM,
 314                 __stringify(KBASE_TL_DEL_ATOM),
 315                 "atom is destroyed",
 316                 "@p",
 317                 "atom"
 318         },
 319         {
 320                 KBASE_TL_LIFELINK_LPU_GPU,
 321                 __stringify(KBASE_TL_LIFELINK_LPU_GPU),
 322                 "lpu is deleted with gpu",
 323                 "@pp",
 324                 "lpu,gpu"
 325         },
 326         {
 327                 KBASE_TL_LIFELINK_AS_GPU,
 328                 __stringify(KBASE_TL_LIFELINK_AS_GPU),
 329                 "address space is deleted with gpu",
 330                 "@pp",
 331                 "address_space,gpu"
 332         },
 333         {
 334                 KBASE_TL_RET_CTX_LPU,
 335                 __stringify(KBASE_TL_RET_CTX_LPU),
 336                 "context is retained by lpu",
 337                 "@pp",
 338                 "ctx,lpu"
 339         },
 340         {
 341                 KBASE_TL_RET_ATOM_CTX,
 342                 __stringify(KBASE_TL_RET_ATOM_CTX),
 343                 "atom is retained by context",
 344                 "@pp",
 345                 "atom,ctx"
 346         },
 347         {
 348                 KBASE_TL_RET_ATOM_LPU,
 349                 __stringify(KBASE_TL_RET_ATOM_LPU),
 350                 "atom is retained by lpu",
 351                 "@pps",
 352                 "atom,lpu,attrib_match_list"
 353         },
 354         {
 355                 KBASE_TL_NRET_CTX_LPU,
 356                 __stringify(KBASE_TL_NRET_CTX_LPU),
 357                 "context is released by lpu",
 358                 "@pp",
 359                 "ctx,lpu"
 360         },
 361         {
 362                 KBASE_TL_NRET_ATOM_CTX,
 363                 __stringify(KBASE_TL_NRET_ATOM_CTX),
 364                 "atom is released by context",
 365                 "@pp",
 366                 "atom,ctx"
 367         },
 368         {
 369                 KBASE_TL_NRET_ATOM_LPU,
 370                 __stringify(KBASE_TL_NRET_ATOM_LPU),
 371                 "atom is released by lpu",
 372                 "@pp",
 373                 "atom,lpu"
 374         },
 375         {
 376                 KBASE_TL_RET_AS_CTX,
 377                 __stringify(KBASE_TL_RET_AS_CTX),
 378                 "address space is retained by context",
 379                 "@pp",
 380                 "address_space,ctx"
 381         },
 382         {
 383                 KBASE_TL_NRET_AS_CTX,
 384                 __stringify(KBASE_TL_NRET_AS_CTX),
 385                 "address space is released by context",
 386                 "@pp",
 387                 "address_space,ctx"
 388         },
 389         {
 390                 KBASE_TL_RET_ATOM_AS,
 391                 __stringify(KBASE_TL_RET_ATOM_AS),
 392                 "atom is retained by address space",
 393                 "@pp",
 394                 "atom,address_space"
 395         },
 396         {
 397                 KBASE_TL_NRET_ATOM_AS,
 398                 __stringify(KBASE_TL_NRET_ATOM_AS),
 399                 "atom is released by address space",
 400                 "@pp",
 401                 "atom,address_space"
 402         },
 403         {
 404                 KBASE_TL_DEP_ATOM_ATOM,
 405                 __stringify(KBASE_TL_DEP_ATOM_ATOM),
 406                 "atom2 depends on atom1",
 407                 "@pp",
 408                 "atom1,atom2"
 409         },
 410         {
 411                 KBASE_TL_ATTRIB_ATOM_CONFIG,
 412                 __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG),
 413                 "atom job slot attributes",
 414                 "@pLLI",
 415                 "atom,descriptor,affinity,config"
 416         },
 417         {
 418                 KBASE_TL_ATTRIB_AS_CONFIG,
 419                 __stringify(KBASE_TL_ATTRIB_AS_CONFIG),
 420                 "address space attributes",
 421                 "@pLLL",
 422                 "address_space,transtab,memattr,transcfg"
 423         },
 424         {
 425                 KBASE_JD_GPU_SOFT_RESET,
 426                 __stringify(KBASE_JD_GPU_SOFT_RESET),
 427                 "gpu soft reset",
 428                 "@p",
 429                 "gpu"
 430         },
 431 };
 432
 433 /* Descriptors of timeline messages transmitted in auxiliary events stream. */
 434 static const struct tp_desc tp_desc_aux[] = {
 435         {
 436                 KBASE_AUX_PM_STATE,
 437                 __stringify(KBASE_AUX_PM_STATE),
 438                 "PM state",
 439                 "@IL",
 440                 "core_type,core_state_bitset"
 441         },
 442         {
 443                 KBASE_AUX_ISSUE_JOB_SOFTSTOP,
 444                 __stringify(KBASE_AUX_ISSUE_JOB_SOFTSTOP),
 445                 "Issuing job soft stop",
 446                 "@p",
 447                 "atom"
 448         },
 449         {
 450                 KBASE_AUX_JOB_SOFTSTOP,
 451                 __stringify(KBASE_AUX_JOB_SOFTSTOP),
 452                 "Job soft stop",
 453                 "@I",
 454                 "tag_id"
 455         },
 456         {
 457                 KBASE_AUX_JOB_SOFTSTOP_EX,
 458                 __stringify(KBASE_AUX_JOB_SOFTSTOP_EX),
 459                 "Job soft stop, more details",
 460                 "@pI",
 461                 "atom,job_type"
 462         },
 463         {
 464                 KBASE_AUX_PAGEFAULT,
 465                 __stringify(KBASE_AUX_PAGEFAULT),
 466                 "Page fault",
 467                 "@IL",
 468                 "ctx_nr,page_cnt_change"
 469         },
 470         {
 471                 KBASE_AUX_PAGESALLOC,
 472                 __stringify(KBASE_AUX_PAGESALLOC),
 473                 "Total alloc pages change",
 474                 "@IL",
 475                 "ctx_nr,page_cnt"
 476         }
 477 };
 478
 479 #if MALI_UNIT_TEST
 480 /* Number of bytes read by user. */
 481 static atomic_t tlstream_bytes_collected = {0};
 482
 483 /* Number of bytes generated by tracepoint messages. */
 484 static atomic_t tlstream_bytes_generated = {0};
 485 #endif /* MALI_UNIT_TEST */
 486
 487 /*****************************************************************************/
 488
 489 /* Indicator of whether the timeline stream file descriptor is used. */
 490 atomic_t kbase_tlstream_enabled = {0};
 491
 492 /*****************************************************************************/
 493
 494 /**
 495  * kbasep_tlstream_get_timestamp - return timestamp
 496  *
 497  * Function returns timestamp value based on raw monotonic timer. Value will
 498  * wrap around zero in case of overflow.
 499  * Return: timestamp value
 500  */
 501 static u64 kbasep_tlstream_get_timestamp(void)
 502 {
 503         struct timespec ts;
 504         u64             timestamp;
 505
 506         getrawmonotonic(&ts);
 507         timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
 508         return timestamp;
 509 }
 510
 511 /**
 512  * kbasep_tlstream_write_bytes - write data to message buffer
 513  * @buffer: buffer where data will be written
 514  * @pos:    position in the buffer where to place data
 515  * @bytes:  pointer to buffer holding data
 516  * @len:    length of data to be written
 517  *
 518  * Return: updated position in the buffer
 519  */
 520 static size_t kbasep_tlstream_write_bytes(
 521                 char       *buffer,
 522                 size_t     pos,
 523                 const void *bytes,
 524                 size_t     len)
 525 {
 526         KBASE_DEBUG_ASSERT(buffer);
 527         KBASE_DEBUG_ASSERT(bytes);
 528
 529         memcpy(&buffer[pos], bytes, len);
 530
 531         return pos + len;
 532 }
 533
 534 /**
 535  * kbasep_tlstream_write_string - write string to message buffer
 536  * @buffer:         buffer where data will be written
 537  * @pos:            position in the buffer where to place data
 538  * @string:         pointer to buffer holding the source string
 539  * @max_write_size: number of bytes that can be stored in buffer
 540  *
 541  * Return: updated position in the buffer
 542  */
 543 static size_t kbasep_tlstream_write_string(
 544                 char       *buffer,
 545                 size_t     pos,
 546                 const char *string,
 547                 size_t     max_write_size)
 548 {
 549         u32 string_len;
 550
 551         KBASE_DEBUG_ASSERT(buffer);
 552         KBASE_DEBUG_ASSERT(string);
 553         /* Timeline string consists of at least string length and nul
 554          * terminator. */
 555         KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char));
 556         max_write_size -= sizeof(string_len);
 557
 558         string_len = strlcpy(
 559                         &buffer[pos + sizeof(string_len)],
 560                         string,
 561                         max_write_size);
 562         string_len += sizeof(char);
 563
 564         /* Make sure that the source string fit into the buffer. */
 565         KBASE_DEBUG_ASSERT(string_len <= max_write_size);
 566
 567         /* Update string length. */
 568         memcpy(&buffer[pos], &string_len, sizeof(string_len));
 569
 570         return pos + sizeof(string_len) + string_len;
 571 }
 572
 573 /**
 574  * kbasep_tlstream_write_timestamp - write timestamp to message buffer
 575  * @buffer: buffer where data will be written
 576  * @pos:    position in the buffer where to place data
 577  *
 578  * Return: updated position in the buffer
 579  */
 580 static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos)
 581 {
 582         u64 timestamp = kbasep_tlstream_get_timestamp();
 583
 584         return kbasep_tlstream_write_bytes(
 585                         buffer, pos,
 586                         &timestamp, sizeof(timestamp));
 587 }
 588
 589 /**
 590  * kbasep_tlstream_put_bits - put bits in a word
 591  * @word:   pointer to the words being modified
 592  * @value:  value that shall be written to given position
 593  * @bitpos: position where value shall be written (in bits)
 594  * @bitlen: length of value (in bits)
 595  */
 596 static void kbasep_tlstream_put_bits(
 597                 u32          *word,
 598                 u32          value,
 599                 unsigned int bitpos,
 600                 unsigned int bitlen)
 601 {
 602         const u32 mask = ((1 << bitlen) - 1) << bitpos;
 603
 604         KBASE_DEBUG_ASSERT(word);
 605         KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen));
 606         KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32);
 607
 608         *word &= ~mask;
 609         *word |= ((value << bitpos) & mask);
 610 }
 611
 612 /**
 613  * kbasep_tlstream_packet_header_setup - setup the packet header
 614  * @buffer:     pointer to the buffer
 615  * @pkt_family: packet's family
 616  * @pkt_type:   packet's type
 617  * @pkt_class:  packet's class
 618  * @stream_id:  stream id
 619  * @numbered:   non-zero if this stream is numbered
 620  *
 621  * Function sets up immutable part of packet header in the given buffer.
 622  */
 623 static void kbasep_tlstream_packet_header_setup(
 624                 char                  *buffer,
 625                 enum tl_packet_family pkt_family,
 626                 enum tl_packet_class  pkt_class,
 627                 enum tl_packet_type   pkt_type,
 628                 unsigned int          stream_id,
 629                 int                   numbered)
 630 {
 631         u32 word0 = 0;
 632         u32 word1 = 0;
 633
 634         KBASE_DEBUG_ASSERT(buffer);
 635         KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL);
 636         KBASE_DEBUG_ASSERT(
 637                         (pkt_type == TL_PACKET_TYPE_HEADER)  ||
 638                         (pkt_type == TL_PACKET_TYPE_SUMMARY) ||
 639                         (pkt_type == TL_PACKET_TYPE_BODY));
 640         KBASE_DEBUG_ASSERT(
 641                         (pkt_class == TL_PACKET_CLASS_OBJ) ||
 642                         (pkt_class == TL_PACKET_CLASS_AUX));
 643
 644         kbasep_tlstream_put_bits(
 645                         &word0, pkt_family,
 646                         PACKET_FAMILY_POS, PACKET_FAMILY_LEN);
 647         kbasep_tlstream_put_bits(
 648                         &word0, pkt_class,
 649                         PACKET_CLASS_POS, PACKET_CLASS_LEN);
 650         kbasep_tlstream_put_bits(
 651                         &word0, pkt_type,
 652                         PACKET_TYPE_POS, PACKET_TYPE_LEN);
 653         kbasep_tlstream_put_bits(
 654                         &word0, stream_id,
 655                         PACKET_STREAMID_POS, PACKET_STREAMID_LEN);
 656
 657         if (numbered)
 658                 kbasep_tlstream_put_bits(
 659                                 &word1, 1,
 660                                 PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN);
 661
 662         memcpy(&buffer[0],             &word0, sizeof(word0));
 663         memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
 664 }
 665
 666 /**
 667  * kbasep_tlstream_packet_header_update - update the packet header
 668  * @buffer:    pointer to the buffer
 669  * @data_size: amount of data carried in this packet
 670  *
 671  * Function updates mutable part of packet header in the given buffer.
 672  * Note that value of data_size must not including size of the header.
 673  */
 674 static void kbasep_tlstream_packet_header_update(
 675                 char   *buffer,
 676                 size_t data_size)
 677 {
 678         u32 word0;
 679         u32 word1;
 680
 681         KBASE_DEBUG_ASSERT(buffer);
 682         CSTD_UNUSED(word0);
 683
 684         memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1));
 685
 686         kbasep_tlstream_put_bits(
 687                         &word1, data_size,
 688                         PACKET_LENGTH_POS, PACKET_LENGTH_LEN);
 689
 690         memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
 691 }
 692
 693 /**
 694  * kbasep_tlstream_packet_number_update - update the packet number
 695  * @buffer:  pointer to the buffer
 696  * @counter: value of packet counter for this packet's stream
 697  *
 698  * Function updates packet number embedded within the packet placed in the
 699  * given buffer.
 700  */
 701 static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter)
 702 {
 703         KBASE_DEBUG_ASSERT(buffer);
 704
 705         memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter));
 706 }
 707
 708 /**
 709  * kbasep_timeline_stream_reset - reset stream
 710  * @stream:  pointer to the stream structure
 711  *
 712  * Function discards all pending messages and resets packet counters.
 713  */
 714 static void kbasep_timeline_stream_reset(struct tl_stream *stream)
 715 {
 716         unsigned int i;
 717
 718         for (i = 0; i < PACKET_COUNT; i++) {
 719                 if (stream->numbered)
 720                         atomic_set(
 721                                         &stream->buffer[i].size,
 722                                         PACKET_HEADER_SIZE +
 723                                         PACKET_NUMBER_SIZE);
 724                 else
 725                         atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE);
 726         }
 727
 728         atomic_set(&stream->wbi, 0);
 729         atomic_set(&stream->rbi, 0);
 730 }
 731
 732 /**
 733  * kbasep_timeline_stream_init - initialize timeline stream
 734  * @stream:      pointer to the stream structure
 735  * @stream_type: stream type
 736  */
 737 static void kbasep_timeline_stream_init(
 738                 struct tl_stream    *stream,
 739                 enum tl_stream_type stream_type)
 740 {
 741         unsigned int i;
 742
 743         KBASE_DEBUG_ASSERT(stream);
 744         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 745
 746         spin_lock_init(&stream->lock);
 747
 748         /* All packets carrying tracepoints shall be numbered. */
 749         if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type)
 750                 stream->numbered = 1;
 751         else
 752                 stream->numbered = 0;
 753
 754         for (i = 0; i < PACKET_COUNT; i++)
 755                 kbasep_tlstream_packet_header_setup(
 756                                 stream->buffer[i].data,
 757                                 tl_stream_cfg[stream_type].pkt_family,
 758                                 tl_stream_cfg[stream_type].pkt_class,
 759                                 tl_stream_cfg[stream_type].pkt_type,
 760                                 tl_stream_cfg[stream_type].stream_id,
 761                                 stream->numbered);
 762
 763         kbasep_timeline_stream_reset(tl_stream[stream_type]);
 764 }
 765
 766 /**
 767  * kbasep_timeline_stream_term - terminate timeline stream
 768  * @stream: pointer to the stream structure
 769  */
 770 static void kbasep_timeline_stream_term(struct tl_stream *stream)
 771 {
 772         KBASE_DEBUG_ASSERT(stream);
 773 }
 774
 775 /**
 776  * kbasep_tlstream_msgbuf_submit - submit packet to the user space
 777  * @stream:     pointer to the stream structure
 778  * @wb_idx_raw: write buffer index
 779  * @wb_size:    length of data stored in current buffer
 780  *
 781  * Function updates currently written buffer with packet header. Then write
 782  * index is incremented and buffer is handled to user space. Parameters
 783  * of new buffer are returned using provided arguments.
 784  *
 785  * Return: length of data in new buffer
 786  *
 787  * Warning:  User must update the stream structure with returned value.
 788  */
 789 static size_t kbasep_tlstream_msgbuf_submit(
 790                 struct tl_stream *stream,
 791                 unsigned int      wb_idx_raw,
 792                 unsigned int      wb_size)
 793 {
 794         unsigned int rb_idx_raw = atomic_read(&stream->rbi);
 795         unsigned int wb_idx = wb_idx_raw % PACKET_COUNT;
 796
 797         /* Set stream as flushed. */
 798         atomic_set(&stream->autoflush_counter, -1);
 799
 800         kbasep_tlstream_packet_header_update(
 801                         stream->buffer[wb_idx].data,
 802                         wb_size - PACKET_HEADER_SIZE);
 803
 804         if (stream->numbered)
 805                 kbasep_tlstream_packet_number_update(
 806                                 stream->buffer[wb_idx].data,
 807                                 wb_idx_raw);
 808
 809         /* Increasing write buffer index will expose this packet to the reader.
 810          * As stream->lock is not taken on reader side we must make sure memory
 811          * is updated correctly before this will happen. */
 812         smp_wmb();
 813         wb_idx_raw++;
 814         atomic_set(&stream->wbi, wb_idx_raw);
 815
 816         /* Inform user that packets are ready for reading. */
 817         wake_up_interruptible(&tl_event_queue);
 818
 819         /* Detect and mark overflow in this stream. */
 820         if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) {
 821                 /* Reader side depends on this increment to correctly handle
 822                  * overflows. The value shall be updated only if it was not
 823                  * modified by the reader. The data holding buffer will not be
 824                  * updated before stream->lock is released, however size of the
 825                  * buffer will. Make sure this increment is globally visible
 826                  * before information about selected write buffer size. */
 827                 atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1);
 828         }
 829
 830         wb_size = PACKET_HEADER_SIZE;
 831         if (stream->numbered)
 832                 wb_size += PACKET_NUMBER_SIZE;
 833
 834         return wb_size;
 835 }
 836
 837 /**
 838  * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer
 839  * @stream_type: type of the stream that shall be locked
 840  * @msg_size:    message size
 841  * @flags:       pointer to store flags passed back on stream release
 842  *
 843  * Function will lock the stream and reserve the number of bytes requested
 844  * in msg_size for the user.
 845  *
 846  * Return: pointer to the buffer where message can be stored
 847  *
 848  * Warning: Stream must be released with kbasep_tlstream_msgbuf_release().
 849  *          Only atomic operations are allowed while stream is locked
 850  *          (i.e. do not use any operation that may sleep).
 851  */
 852 static char *kbasep_tlstream_msgbuf_acquire(
 853                 enum tl_stream_type stream_type,
 854                 size_t              msg_size,
 855                 unsigned long       *flags) __acquires(&stream->lock)
 856 {
 857         struct tl_stream *stream;
 858         unsigned int     wb_idx_raw;
 859         unsigned int     wb_idx;
 860         size_t           wb_size;
 861
 862         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 863         KBASE_DEBUG_ASSERT(
 864                         PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >=
 865                         msg_size);
 866
 867         stream = tl_stream[stream_type];
 868
 869         spin_lock_irqsave(&stream->lock, *flags);
 870
 871         wb_idx_raw = atomic_read(&stream->wbi);
 872         wb_idx     = wb_idx_raw % PACKET_COUNT;
 873         wb_size    = atomic_read(&stream->buffer[wb_idx].size);
 874
 875         /* Select next buffer if data will not fit into current one. */
 876         if (PACKET_SIZE < wb_size + msg_size) {
 877                 wb_size = kbasep_tlstream_msgbuf_submit(
 878                                 stream, wb_idx_raw, wb_size);
 879                 wb_idx  = (wb_idx_raw + 1) % PACKET_COUNT;
 880         }
 881
 882         /* Reserve space in selected buffer. */
 883         atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size);
 884
 885 #if MALI_UNIT_TEST
 886         atomic_add(msg_size, &tlstream_bytes_generated);
 887 #endif /* MALI_UNIT_TEST */
 888
 889         return &stream->buffer[wb_idx].data[wb_size];
 890 }
 891
 892 /**
 893  * kbasep_tlstream_msgbuf_release - unlock selected stream
 894  * @stream_type:  type of the stream that shall be locked
 895  * @flags:        value obtained during stream acquire
 896  *
 897  * Function releases stream that has been previously locked with a call to
 898  * kbasep_tlstream_msgbuf_acquire().
 899  */
 900 static void kbasep_tlstream_msgbuf_release(
 901                 enum tl_stream_type stream_type,
 902                 unsigned long       flags) __releases(&stream->lock)
 903 {
 904         struct tl_stream *stream;
 905
 906         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
 907
 908         stream = tl_stream[stream_type];
 909
 910         /* Mark stream as containing unflushed data. */
 911         atomic_set(&stream->autoflush_counter, 0);
 912
 913         spin_unlock_irqrestore(&stream->lock, flags);
 914 }
 915
 916 /*****************************************************************************/
 917
 918 /**
 919  * kbasep_tlstream_flush_stream - flush stream
 920  * @stype:  type of stream to be flushed
 921  *
 922  * Flush pending data in timeline stream.
 923  */
 924 static void kbasep_tlstream_flush_stream(enum tl_stream_type stype)
 925 {
 926         struct tl_stream *stream = tl_stream[stype];
 927         unsigned long    flags;
 928         unsigned int     wb_idx_raw;
 929         unsigned int     wb_idx;
 930         size_t           wb_size;
 931         size_t           min_size = PACKET_HEADER_SIZE;
 932
 933         if (stream->numbered)
 934                 min_size += PACKET_NUMBER_SIZE;
 935
 936         spin_lock_irqsave(&stream->lock, flags);
 937
 938         wb_idx_raw = atomic_read(&stream->wbi);
 939         wb_idx     = wb_idx_raw % PACKET_COUNT;
 940         wb_size    = atomic_read(&stream->buffer[wb_idx].size);
 941
 942         if (wb_size > min_size) {
 943                 wb_size = kbasep_tlstream_msgbuf_submit(
 944                                 stream, wb_idx_raw, wb_size);
 945                 wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
 946                 atomic_set(&stream->buffer[wb_idx].size, wb_size);
 947         }
 948         spin_unlock_irqrestore(&stream->lock, flags);
 949 }
 950
 951 /**
 952  * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback
 953  * @data:  unused
 954  *
 955  * Timer is executed periodically to check if any of the stream contains
 956  * buffer ready to be submitted to user space.
 957  */
 958 static void kbasep_tlstream_autoflush_timer_callback(unsigned long data)
 959 {
 960         enum tl_stream_type stype;
 961         int                 rcode;
 962
 963         CSTD_UNUSED(data);
 964
 965         for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) {
 966                 struct tl_stream *stream = tl_stream[stype];
 967                 unsigned long    flags;
 968                 unsigned int     wb_idx_raw;
 969                 unsigned int     wb_idx;
 970                 size_t           wb_size;
 971                 size_t           min_size = PACKET_HEADER_SIZE;
 972
 973                 int af_cnt = atomic_read(&stream->autoflush_counter);
 974
 975                 /* Check if stream contain unflushed data. */
 976                 if (0 > af_cnt)
 977                         continue;
 978
 979                 /* Check if stream should be flushed now. */
 980                 if (af_cnt != atomic_cmpxchg(
 981                                         &stream->autoflush_counter,
 982                                         af_cnt,
 983                                         af_cnt + 1))
 984                         continue;
 985                 if (!af_cnt)
 986                         continue;
 987
 988                 /* Autoflush this stream. */
 989                 if (stream->numbered)
 990                         min_size += PACKET_NUMBER_SIZE;
 991
 992                 spin_lock_irqsave(&stream->lock, flags);
 993
 994                 wb_idx_raw = atomic_read(&stream->wbi);
 995                 wb_idx     = wb_idx_raw % PACKET_COUNT;
 996                 wb_size    = atomic_read(&stream->buffer[wb_idx].size);
 997
 998                 if (wb_size > min_size) {
 999                         wb_size = kbasep_tlstream_msgbuf_submit(
1000                                         stream, wb_idx_raw, wb_size);
1001                         wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
1002                         atomic_set(&stream->buffer[wb_idx].size,
1003                                         wb_size);
1004                 }
1005                 spin_unlock_irqrestore(&stream->lock, flags);
1006         }
1007
1008         if (atomic_read(&autoflush_timer_active))
1009                 rcode = mod_timer(
1010                                 &autoflush_timer,
1011                                 jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
1012         CSTD_UNUSED(rcode);
1013 }
1014
1015 /**
1016  * kbasep_tlstream_packet_pending - check timeline streams for pending packets
1017  * @stype:      pointer to variable where stream type will be placed
1018  * @rb_idx_raw: pointer to variable where read buffer index will be placed
1019  *
1020  * Function checks all streams for pending packets. It will stop as soon as
1021  * packet ready to be submitted to user space is detected. Variables under
1022  * pointers, passed as the parameters to this function will be updated with
1023  * values pointing to right stream and buffer.
1024  *
1025  * Return: non-zero if any of timeline streams has at last one packet ready
1026  */
1027 static int kbasep_tlstream_packet_pending(
1028                 enum tl_stream_type *stype,
1029                 unsigned int        *rb_idx_raw)
1030 {
1031         int pending = 0;
1032
1033         KBASE_DEBUG_ASSERT(stype);
1034         KBASE_DEBUG_ASSERT(rb_idx_raw);
1035
1036         for (
1037                         *stype = 0;
1038                         (*stype < TL_STREAM_TYPE_COUNT) && !pending;
1039                         (*stype)++) {
1040                 if (NULL != tl_stream[*stype]) {
1041                         *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi);
1042                         /* Read buffer index may be updated by writer in case of
1043                          * overflow. Read and write buffer indexes must be
1044                          * loaded in correct order. */
1045                         smp_rmb();
1046                         if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw)
1047                                 pending = 1;
1048                 }
1049         }
1050         (*stype)--;
1051
1052         return pending;
1053 }
1054
1055 /**
1056  * kbasep_tlstream_read - copy data from streams to buffer provided by user
1057  * @filp:   pointer to file structure (unused)
1058  * @buffer: pointer to the buffer provided by user
1059  * @size:   maximum amount of data that can be stored in the buffer
1060  * @f_pos:  pointer to file offset (unused)
1061  *
1062  * Return: number of bytes stored in the buffer
1063  */
1064 static ssize_t kbasep_tlstream_read(
1065                 struct file *filp,
1066                 char __user *buffer,
1067                 size_t      size,
1068                 loff_t      *f_pos)
1069 {
1070         ssize_t copy_len = 0;
1071
1072         KBASE_DEBUG_ASSERT(filp);
1073         KBASE_DEBUG_ASSERT(buffer);
1074         KBASE_DEBUG_ASSERT(f_pos);
1075         CSTD_UNUSED(filp);
1076
1077         if ((0 > *f_pos) || (PACKET_SIZE > size))
1078                 return -EINVAL;
1079
1080         mutex_lock(&tl_reader_lock);
1081
1082         while (copy_len < size) {
1083                 enum tl_stream_type stype;
1084                 unsigned int        rb_idx_raw;
1085                 unsigned int        rb_idx;
1086                 size_t              rb_size;
1087
1088                 /* If we don't have any data yet, wait for packet to be
1089                  * submitted. If we already read some packets and there is no
1090                  * packet pending return back to user. */
1091                 if (0 < copy_len) {
1092                         if (!kbasep_tlstream_packet_pending(
1093                                                 &stype,
1094                                                 &rb_idx_raw))
1095                                 break;
1096                 } else {
1097                         if (wait_event_interruptible(
1098                                                 tl_event_queue,
1099                                                 kbasep_tlstream_packet_pending(
1100                                                         &stype,
1101                                                         &rb_idx_raw))) {
1102                                 copy_len = -ERESTARTSYS;
1103                                 break;
1104                         }
1105                 }
1106
1107                 /* Check if this packet fits into the user buffer.
1108                  * If so copy its content. */
1109                 rb_idx = rb_idx_raw % PACKET_COUNT;
1110                 rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size);
1111                 if (rb_size > size - copy_len)
1112                         break;
1113                 if (copy_to_user(
1114                                         &buffer[copy_len],
1115                                         tl_stream[stype]->buffer[rb_idx].data,
1116                                         rb_size)) {
1117                         copy_len = -EFAULT;
1118                         break;
1119                 }
1120
1121                 /* If the rbi still points to the packet we just processed
1122                  * then there was no overflow so we add the copied size to
1123                  * copy_len and move rbi on to the next packet
1124                  */
1125                 smp_rmb();
1126                 if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) {
1127                         copy_len += rb_size;
1128                         atomic_inc(&tl_stream[stype]->rbi);
1129
1130 #if MALI_UNIT_TEST
1131                         atomic_add(rb_size, &tlstream_bytes_collected);
1132 #endif /* MALI_UNIT_TEST */
1133                 }
1134         }
1135
1136         mutex_unlock(&tl_reader_lock);
1137
1138         return copy_len;
1139 }
1140
1141 /**
1142  * kbasep_tlstream_poll - poll timeline stream for packets
1143  * @filp: pointer to file structure
1144  * @wait: pointer to poll table
1145  * Return: POLLIN if data can be read without blocking, otherwise zero
1146  */
1147 static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait)
1148 {
1149         enum tl_stream_type stream_type;
1150         unsigned int        rb_idx;
1151
1152         KBASE_DEBUG_ASSERT(filp);
1153         KBASE_DEBUG_ASSERT(wait);
1154
1155         poll_wait(filp, &tl_event_queue, wait);
1156         if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx))
1157                 return POLLIN;
1158         return 0;
1159 }
1160
1161 /**
1162  * kbasep_tlstream_release - release timeline stream descriptor
1163  * @inode: pointer to inode structure
1164  * @filp:  pointer to file structure
1165  *
1166  * Return always return zero
1167  */
1168 static int kbasep_tlstream_release(struct inode *inode, struct file *filp)
1169 {
1170         KBASE_DEBUG_ASSERT(inode);
1171         KBASE_DEBUG_ASSERT(filp);
1172         CSTD_UNUSED(inode);
1173         CSTD_UNUSED(filp);
1174
1175         /* Stop autoflush timer before releasing access to streams. */
1176         atomic_set(&autoflush_timer_active, 0);
1177         del_timer_sync(&autoflush_timer);
1178
1179         atomic_set(&kbase_tlstream_enabled, 0);
1180         return 0;
1181 }
1182
1183 /**
1184  * kbasep_tlstream_timeline_header - prepare timeline header stream packet
1185  * @stream_type: type of the stream that will carry header data
1186  * @tp_desc:     pointer to array with tracepoint descriptors
1187  * @tp_count:    number of descriptors in the given array
1188  *
1189  * Functions fills in information about tracepoints stored in body stream
1190  * associated with this header stream.
1191  */
1192 static void kbasep_tlstream_timeline_header(
1193                 enum tl_stream_type  stream_type,
1194                 const struct tp_desc *tp_desc,
1195                 u32                  tp_count)
1196 {
1197         const u8      tv = SWTRACE_VERSION; /* protocol version */
1198         const u8      ps = sizeof(void *); /* pointer size */
1199         size_t        msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count);
1200         char          *buffer;
1201         size_t        pos = 0;
1202         unsigned long flags;
1203         unsigned int  i;
1204
1205         KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
1206         KBASE_DEBUG_ASSERT(tp_desc);
1207
1208         /* Calculate the size of the timeline message. */
1209         for (i = 0; i < tp_count; i++) {
1210                 msg_size += sizeof(tp_desc[i].id);
1211                 msg_size +=
1212                         strnlen(tp_desc[i].id_str,    STRLEN_MAX) +
1213                         sizeof(char) + sizeof(u32);
1214                 msg_size +=
1215                         strnlen(tp_desc[i].name,      STRLEN_MAX) +
1216                         sizeof(char) + sizeof(u32);
1217                 msg_size +=
1218                         strnlen(tp_desc[i].arg_types, STRLEN_MAX) +
1219                         sizeof(char) + sizeof(u32);
1220                 msg_size +=
1221                         strnlen(tp_desc[i].arg_names, STRLEN_MAX) +
1222                         sizeof(char) + sizeof(u32);
1223         }
1224
1225         KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size);
1226
1227         buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags);
1228         KBASE_DEBUG_ASSERT(buffer);
1229
1230         pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv));
1231         pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps));
1232         pos = kbasep_tlstream_write_bytes(
1233                         buffer, pos, &tp_count, sizeof(tp_count));
1234
1235         for (i = 0; i < tp_count; i++) {
1236                 pos = kbasep_tlstream_write_bytes(
1237                                 buffer, pos,
1238                                 &tp_desc[i].id, sizeof(tp_desc[i].id));
1239                 pos = kbasep_tlstream_write_string(
1240                                 buffer, pos,
1241                                 tp_desc[i].id_str, msg_size - pos);
1242                 pos = kbasep_tlstream_write_string(
1243                                 buffer, pos,
1244                                 tp_desc[i].name, msg_size - pos);
1245                 pos = kbasep_tlstream_write_string(
1246                                 buffer, pos,
1247                                 tp_desc[i].arg_types, msg_size - pos);
1248                 pos = kbasep_tlstream_write_string(
1249                                 buffer, pos,
1250                                 tp_desc[i].arg_names, msg_size - pos);
1251         }
1252
1253         KBASE_DEBUG_ASSERT(msg_size == pos);
1254
1255         kbasep_tlstream_msgbuf_release(stream_type, flags);
1256
1257         /* We don't expect any more data to be read in this stream.
1258          * As header stream must be read before its associated body stream,
1259          * make this packet visible to the user straightaway. */
1260         kbasep_tlstream_flush_stream(stream_type);
1261 }
1262
1263 /*****************************************************************************/
1264
1265 int kbase_tlstream_init(void)
1266 {
1267         enum tl_stream_type i;
1268
1269         /* Prepare stream structures. */
1270         for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
1271                 tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL);
1272                 if (!tl_stream[i])
1273                         break;
1274                 kbasep_timeline_stream_init(tl_stream[i], i);
1275         }
1276         if (TL_STREAM_TYPE_COUNT > i) {
1277                 for (; i > 0; i--) {
1278                         kbasep_timeline_stream_term(tl_stream[i - 1]);
1279                         kfree(tl_stream[i - 1]);
1280                 }
1281                 return -ENOMEM;
1282         }
1283
1284         /* Initialize autoflush timer. */
1285         atomic_set(&autoflush_timer_active, 0);
1286         setup_timer(&autoflush_timer,
1287                         kbasep_tlstream_autoflush_timer_callback,
1288                         0);
1289
1290         return 0;
1291 }
1292
1293 void kbase_tlstream_term(void)
1294 {
1295         enum tl_stream_type i;
1296
1297         for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
1298                 kbasep_timeline_stream_term(tl_stream[i]);
1299                 kfree(tl_stream[i]);
1300         }
1301 }
1302
1303 int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd)
1304 {
1305         if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, 1)) {
1306                 int rcode;
1307
1308                 *fd = anon_inode_getfd(
1309                                 "[mali_tlstream]",
1310                                 &kbasep_tlstream_fops,
1311                                 kctx,
1312                                 O_RDONLY | O_CLOEXEC);
1313                 if (0 > *fd) {
1314                         atomic_set(&kbase_tlstream_enabled, 0);
1315                         return *fd;
1316                 }
1317
1318                 /* Reset and initialize header streams. */
1319                 kbasep_timeline_stream_reset(
1320                                 tl_stream[TL_STREAM_TYPE_OBJ_HEADER]);
1321                 kbasep_timeline_stream_reset(
1322                                 tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]);
1323                 kbasep_timeline_stream_reset(
1324                                 tl_stream[TL_STREAM_TYPE_AUX_HEADER]);
1325                 kbasep_tlstream_timeline_header(
1326                                 TL_STREAM_TYPE_OBJ_HEADER,
1327                                 tp_desc_obj,
1328                                 ARRAY_SIZE(tp_desc_obj));
1329                 kbasep_tlstream_timeline_header(
1330                                 TL_STREAM_TYPE_AUX_HEADER,
1331                                 tp_desc_aux,
1332                                 ARRAY_SIZE(tp_desc_aux));
1333
1334                 /* Start autoflush timer. */
1335                 atomic_set(&autoflush_timer_active, 1);
1336                 rcode = mod_timer(
1337                                 &autoflush_timer,
1338                                 jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
1339                 CSTD_UNUSED(rcode);
1340
1341         } else {
1342                 *fd = -EBUSY;
1343         }
1344
1345         return 0;
1346 }
1347
1348 void kbase_tlstream_flush_streams(void)
1349 {
1350         enum tl_stream_type stype;
1351
1352         for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++)
1353                 kbasep_tlstream_flush_stream(stype);
1354 }
1355
1356 void kbase_tlstream_reset_body_streams(void)
1357 {
1358         kbasep_timeline_stream_reset(
1359                         tl_stream[TL_STREAM_TYPE_OBJ]);
1360         kbasep_timeline_stream_reset(
1361                         tl_stream[TL_STREAM_TYPE_AUX]);
1362 }
1363
1364 #if MALI_UNIT_TEST
1365 void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated)
1366 {
1367         KBASE_DEBUG_ASSERT(bytes_collected);
1368         KBASE_DEBUG_ASSERT(bytes_generated);
1369         *bytes_collected = atomic_read(&tlstream_bytes_collected);
1370         *bytes_generated = atomic_read(&tlstream_bytes_generated);
1371 }
1372 #endif /* MALI_UNIT_TEST */
1373
1374 /*****************************************************************************/
1375
1376 void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid)
1377 {
1378         const u32     msg_id = KBASE_TL_NEW_CTX;
1379         const size_t  msg_size =
1380                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) +
1381                 sizeof(tgid);
1382         unsigned long flags;
1383         char          *buffer;
1384         size_t        pos = 0;
1385
1386         buffer = kbasep_tlstream_msgbuf_acquire(
1387                         TL_STREAM_TYPE_OBJ_SUMMARY,
1388                         msg_size, &flags);
1389         KBASE_DEBUG_ASSERT(buffer);
1390
1391         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1392         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1393         pos = kbasep_tlstream_write_bytes(
1394                         buffer, pos, &context, sizeof(context));
1395         pos = kbasep_tlstream_write_bytes(
1396                         buffer, pos, &nr, sizeof(nr));
1397         pos = kbasep_tlstream_write_bytes(
1398                         buffer, pos, &tgid, sizeof(tgid));
1399
1400         KBASE_DEBUG_ASSERT(msg_size == pos);
1401
1402         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1403 }
1404
1405 void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count)
1406 {
1407         const u32     msg_id = KBASE_TL_NEW_GPU;
1408         const size_t  msg_size =
1409                 sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) +
1410                 sizeof(core_count);
1411         unsigned long flags;
1412         char          *buffer;
1413         size_t        pos = 0;
1414
1415         buffer = kbasep_tlstream_msgbuf_acquire(
1416                         TL_STREAM_TYPE_OBJ_SUMMARY,
1417                         msg_size, &flags);
1418         KBASE_DEBUG_ASSERT(buffer);
1419
1420         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1421         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1422         pos = kbasep_tlstream_write_bytes(
1423                         buffer, pos, &gpu, sizeof(gpu));
1424         pos = kbasep_tlstream_write_bytes(
1425                         buffer, pos, &id, sizeof(id));
1426         pos = kbasep_tlstream_write_bytes(
1427                         buffer, pos, &core_count, sizeof(core_count));
1428         KBASE_DEBUG_ASSERT(msg_size == pos);
1429
1430         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1431 }
1432
1433 void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn)
1434 {
1435         const u32     msg_id = KBASE_TL_NEW_LPU;
1436         const size_t  msg_size =
1437                 sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) +
1438                 sizeof(fn);
1439         unsigned long flags;
1440         char          *buffer;
1441         size_t        pos = 0;
1442
1443         buffer = kbasep_tlstream_msgbuf_acquire(
1444                         TL_STREAM_TYPE_OBJ_SUMMARY,
1445                         msg_size, &flags);
1446         KBASE_DEBUG_ASSERT(buffer);
1447
1448         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1449         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1450         pos = kbasep_tlstream_write_bytes(
1451                         buffer, pos, &lpu, sizeof(lpu));
1452         pos = kbasep_tlstream_write_bytes(
1453                         buffer, pos, &nr, sizeof(nr));
1454         pos = kbasep_tlstream_write_bytes(
1455                         buffer, pos, &fn, sizeof(fn));
1456         KBASE_DEBUG_ASSERT(msg_size == pos);
1457
1458         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1459 }
1460
1461 void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu)
1462 {
1463         const u32     msg_id = KBASE_TL_LIFELINK_LPU_GPU;
1464         const size_t  msg_size =
1465                 sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu);
1466         unsigned long flags;
1467         char          *buffer;
1468         size_t        pos = 0;
1469
1470         buffer = kbasep_tlstream_msgbuf_acquire(
1471                         TL_STREAM_TYPE_OBJ_SUMMARY,
1472                         msg_size, &flags);
1473         KBASE_DEBUG_ASSERT(buffer);
1474
1475         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1476         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1477         pos = kbasep_tlstream_write_bytes(
1478                         buffer, pos, &lpu, sizeof(lpu));
1479         pos = kbasep_tlstream_write_bytes(
1480                         buffer, pos, &gpu, sizeof(gpu));
1481         KBASE_DEBUG_ASSERT(msg_size == pos);
1482
1483         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1484 }
1485
1486 void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr)
1487 {
1488         const u32     msg_id = KBASE_TL_NEW_AS;
1489         const size_t  msg_size =
1490                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr);
1491         unsigned long flags;
1492         char          *buffer;
1493         size_t        pos = 0;
1494
1495         buffer = kbasep_tlstream_msgbuf_acquire(
1496                         TL_STREAM_TYPE_OBJ_SUMMARY,
1497                         msg_size, &flags);
1498         KBASE_DEBUG_ASSERT(buffer);
1499
1500         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1501         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1502         pos = kbasep_tlstream_write_bytes(
1503                         buffer, pos, &as, sizeof(as));
1504         pos = kbasep_tlstream_write_bytes(
1505                         buffer, pos, &nr, sizeof(nr));
1506         KBASE_DEBUG_ASSERT(msg_size == pos);
1507
1508         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1509 }
1510
1511 void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu)
1512 {
1513         const u32     msg_id = KBASE_TL_LIFELINK_AS_GPU;
1514         const size_t  msg_size =
1515                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu);
1516         unsigned long flags;
1517         char          *buffer;
1518         size_t        pos = 0;
1519
1520         buffer = kbasep_tlstream_msgbuf_acquire(
1521                         TL_STREAM_TYPE_OBJ_SUMMARY,
1522                         msg_size, &flags);
1523         KBASE_DEBUG_ASSERT(buffer);
1524
1525         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1526         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1527         pos = kbasep_tlstream_write_bytes(
1528                         buffer, pos, &as, sizeof(as));
1529         pos = kbasep_tlstream_write_bytes(
1530                         buffer, pos, &gpu, sizeof(gpu));
1531         KBASE_DEBUG_ASSERT(msg_size == pos);
1532
1533         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
1534 }
1535
1536 /*****************************************************************************/
1537
1538 void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid)
1539 {
1540         const u32     msg_id = KBASE_TL_NEW_CTX;
1541         const size_t  msg_size =
1542                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) +
1543                 sizeof(tgid);
1544         unsigned long flags;
1545         char          *buffer;
1546         size_t        pos = 0;
1547
1548         buffer = kbasep_tlstream_msgbuf_acquire(
1549                         TL_STREAM_TYPE_OBJ,
1550                         msg_size, &flags);
1551         KBASE_DEBUG_ASSERT(buffer);
1552
1553         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1554         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1555         pos = kbasep_tlstream_write_bytes(
1556                         buffer, pos, &context, sizeof(context));
1557         pos = kbasep_tlstream_write_bytes(
1558                         buffer, pos, &nr, sizeof(nr));
1559         pos = kbasep_tlstream_write_bytes(
1560                         buffer, pos, &tgid, sizeof(tgid));
1561         KBASE_DEBUG_ASSERT(msg_size == pos);
1562
1563         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1564 }
1565
1566 void __kbase_tlstream_tl_new_atom(void *atom, u32 nr)
1567 {
1568         const u32     msg_id = KBASE_TL_NEW_ATOM;
1569         const size_t  msg_size =
1570                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(nr);
1571         unsigned long flags;
1572         char          *buffer;
1573         size_t        pos = 0;
1574
1575         buffer = kbasep_tlstream_msgbuf_acquire(
1576                         TL_STREAM_TYPE_OBJ,
1577                         msg_size, &flags);
1578         KBASE_DEBUG_ASSERT(buffer);
1579
1580         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1581         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1582         pos = kbasep_tlstream_write_bytes(
1583                         buffer, pos, &atom, sizeof(atom));
1584         pos = kbasep_tlstream_write_bytes(
1585                         buffer, pos, &nr, sizeof(nr));
1586         KBASE_DEBUG_ASSERT(msg_size == pos);
1587
1588         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1589 }
1590
1591 void __kbase_tlstream_tl_del_ctx(void *context)
1592 {
1593         const u32     msg_id = KBASE_TL_DEL_CTX;
1594         const size_t  msg_size =
1595                 sizeof(msg_id) + sizeof(u64) + sizeof(context);
1596         unsigned long flags;
1597         char          *buffer;
1598         size_t        pos = 0;
1599
1600         buffer = kbasep_tlstream_msgbuf_acquire(
1601                         TL_STREAM_TYPE_OBJ,
1602                         msg_size, &flags);
1603         KBASE_DEBUG_ASSERT(buffer);
1604
1605         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1606         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1607         pos = kbasep_tlstream_write_bytes(
1608                         buffer, pos, &context, sizeof(context));
1609         KBASE_DEBUG_ASSERT(msg_size == pos);
1610
1611         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1612 }
1613
1614 void __kbase_tlstream_tl_del_atom(void *atom)
1615 {
1616         const u32     msg_id = KBASE_TL_DEL_ATOM;
1617         const size_t  msg_size =
1618                 sizeof(msg_id) + sizeof(u64) + sizeof(atom);
1619         unsigned long flags;
1620         char          *buffer;
1621         size_t        pos = 0;
1622
1623         buffer = kbasep_tlstream_msgbuf_acquire(
1624                         TL_STREAM_TYPE_OBJ,
1625                         msg_size, &flags);
1626         KBASE_DEBUG_ASSERT(buffer);
1627
1628         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1629         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1630         pos = kbasep_tlstream_write_bytes(
1631                         buffer, pos, &atom, sizeof(atom));
1632         KBASE_DEBUG_ASSERT(msg_size == pos);
1633
1634         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1635 }
1636
1637 void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu)
1638 {
1639         const u32     msg_id = KBASE_TL_RET_CTX_LPU;
1640         const size_t  msg_size =
1641                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu);
1642         unsigned long flags;
1643         char          *buffer;
1644         size_t        pos = 0;
1645
1646         buffer = kbasep_tlstream_msgbuf_acquire(
1647                         TL_STREAM_TYPE_OBJ,
1648                         msg_size, &flags);
1649         KBASE_DEBUG_ASSERT(buffer);
1650
1651         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1652         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1653         pos = kbasep_tlstream_write_bytes(
1654                         buffer, pos, &context, sizeof(context));
1655         pos = kbasep_tlstream_write_bytes(
1656                         buffer, pos, &lpu, sizeof(lpu));
1657         KBASE_DEBUG_ASSERT(msg_size == pos);
1658
1659         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1660 }
1661
1662 void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context)
1663 {
1664         const u32     msg_id = KBASE_TL_RET_ATOM_CTX;
1665         const size_t  msg_size =
1666                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
1667         unsigned long flags;
1668         char          *buffer;
1669         size_t        pos = 0;
1670
1671         buffer = kbasep_tlstream_msgbuf_acquire(
1672                         TL_STREAM_TYPE_OBJ,
1673                         msg_size, &flags);
1674         KBASE_DEBUG_ASSERT(buffer);
1675
1676         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1677         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1678         pos = kbasep_tlstream_write_bytes(
1679                         buffer, pos, &atom, sizeof(atom));
1680         pos = kbasep_tlstream_write_bytes(
1681                         buffer, pos, &context, sizeof(context));
1682         KBASE_DEBUG_ASSERT(msg_size == pos);
1683
1684         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1685 }
1686
1687 void __kbase_tlstream_tl_ret_atom_lpu(
1688                 void *atom, void *lpu, const char *attrib_match_list)
1689 {
1690         const u32     msg_id = KBASE_TL_RET_ATOM_LPU;
1691         const size_t  msg_s0 = sizeof(u32) + sizeof(char) +
1692                         strnlen(attrib_match_list, STRLEN_MAX);
1693         const size_t  msg_size =
1694                         sizeof(msg_id) + sizeof(u64) +
1695                         sizeof(atom) + sizeof(lpu) + msg_s0;
1696         unsigned long flags;
1697         char          *buffer;
1698         size_t        pos = 0;
1699
1700         buffer = kbasep_tlstream_msgbuf_acquire(
1701                         TL_STREAM_TYPE_OBJ,
1702                         msg_size, &flags);
1703         KBASE_DEBUG_ASSERT(buffer);
1704
1705         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1706         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1707         pos = kbasep_tlstream_write_bytes(
1708                         buffer, pos, &atom, sizeof(atom));
1709         pos = kbasep_tlstream_write_bytes(
1710                         buffer, pos, &lpu, sizeof(lpu));
1711         pos = kbasep_tlstream_write_string(
1712                         buffer, pos, attrib_match_list, msg_s0);
1713         KBASE_DEBUG_ASSERT(msg_size == pos);
1714
1715         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1716 }
1717
1718 void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu)
1719 {
1720         const u32     msg_id = KBASE_TL_NRET_CTX_LPU;
1721         const size_t  msg_size =
1722                 sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu);
1723         unsigned long flags;
1724         char          *buffer;
1725         size_t        pos = 0;
1726
1727         buffer = kbasep_tlstream_msgbuf_acquire(
1728                         TL_STREAM_TYPE_OBJ,
1729                         msg_size, &flags);
1730         KBASE_DEBUG_ASSERT(buffer);
1731
1732         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1733         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1734         pos = kbasep_tlstream_write_bytes(
1735                         buffer, pos, &context, sizeof(context));
1736         pos = kbasep_tlstream_write_bytes(
1737                         buffer, pos, &lpu, sizeof(lpu));
1738         KBASE_DEBUG_ASSERT(msg_size == pos);
1739
1740         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1741 }
1742
1743 void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context)
1744 {
1745         const u32     msg_id = KBASE_TL_NRET_ATOM_CTX;
1746         const size_t  msg_size =
1747                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
1748         unsigned long flags;
1749         char          *buffer;
1750         size_t        pos = 0;
1751
1752         buffer = kbasep_tlstream_msgbuf_acquire(
1753                         TL_STREAM_TYPE_OBJ,
1754                         msg_size, &flags);
1755         KBASE_DEBUG_ASSERT(buffer);
1756
1757         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1758         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1759         pos = kbasep_tlstream_write_bytes(
1760                         buffer, pos, &atom, sizeof(atom));
1761         pos = kbasep_tlstream_write_bytes(
1762                         buffer, pos, &context, sizeof(context));
1763         KBASE_DEBUG_ASSERT(msg_size == pos);
1764
1765         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1766 }
1767
1768 void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2)
1769 {
1770         const u32     msg_id = KBASE_TL_DEP_ATOM_ATOM;
1771         const size_t  msg_size =
1772                 sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
1773         unsigned long flags;
1774         char          *buffer;
1775         size_t        pos = 0;
1776
1777         buffer = kbasep_tlstream_msgbuf_acquire(
1778                         TL_STREAM_TYPE_OBJ,
1779                         msg_size, &flags);
1780         KBASE_DEBUG_ASSERT(buffer);
1781
1782         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1783         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1784         pos = kbasep_tlstream_write_bytes(
1785                         buffer, pos, &atom1, sizeof(atom1));
1786         pos = kbasep_tlstream_write_bytes(
1787                         buffer, pos, &atom2, sizeof(atom2));
1788         KBASE_DEBUG_ASSERT(msg_size == pos);
1789
1790         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1791 }
1792
1793 void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu)
1794 {
1795         const u32     msg_id = KBASE_TL_NRET_ATOM_LPU;
1796         const size_t  msg_size =
1797                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
1798         unsigned long flags;
1799         char          *buffer;
1800         size_t        pos = 0;
1801
1802         buffer = kbasep_tlstream_msgbuf_acquire(
1803                         TL_STREAM_TYPE_OBJ,
1804                         msg_size, &flags);
1805         KBASE_DEBUG_ASSERT(buffer);
1806
1807         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1808         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1809         pos = kbasep_tlstream_write_bytes(
1810                         buffer, pos, &atom, sizeof(atom));
1811         pos = kbasep_tlstream_write_bytes(
1812                         buffer, pos, &lpu, sizeof(lpu));
1813         KBASE_DEBUG_ASSERT(msg_size == pos);
1814
1815         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1816 }
1817
1818 void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx)
1819 {
1820         const u32     msg_id = KBASE_TL_RET_AS_CTX;
1821         const size_t  msg_size =
1822                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx);
1823         unsigned long flags;
1824         char          *buffer;
1825         size_t        pos = 0;
1826
1827         buffer = kbasep_tlstream_msgbuf_acquire(
1828                         TL_STREAM_TYPE_OBJ,
1829                         msg_size, &flags);
1830         KBASE_DEBUG_ASSERT(buffer);
1831
1832         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1833         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1834         pos = kbasep_tlstream_write_bytes(
1835                         buffer, pos, &as, sizeof(as));
1836         pos = kbasep_tlstream_write_bytes(
1837                         buffer, pos, &ctx, sizeof(ctx));
1838         KBASE_DEBUG_ASSERT(msg_size == pos);
1839
1840         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1841 }
1842
1843 void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx)
1844 {
1845         const u32     msg_id = KBASE_TL_NRET_AS_CTX;
1846         const size_t  msg_size =
1847                 sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx);
1848         unsigned long flags;
1849         char          *buffer;
1850         size_t        pos = 0;
1851
1852         buffer = kbasep_tlstream_msgbuf_acquire(
1853                         TL_STREAM_TYPE_OBJ,
1854                         msg_size, &flags);
1855         KBASE_DEBUG_ASSERT(buffer);
1856
1857         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1858         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1859         pos = kbasep_tlstream_write_bytes(
1860                         buffer, pos, &as, sizeof(as));
1861         pos = kbasep_tlstream_write_bytes(
1862                         buffer, pos, &ctx, sizeof(ctx));
1863         KBASE_DEBUG_ASSERT(msg_size == pos);
1864
1865         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1866 }
1867
1868 void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as)
1869 {
1870         const u32     msg_id = KBASE_TL_RET_ATOM_AS;
1871         const size_t  msg_size =
1872                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as);
1873         unsigned long flags;
1874         char          *buffer;
1875         size_t        pos = 0;
1876
1877         buffer = kbasep_tlstream_msgbuf_acquire(
1878                         TL_STREAM_TYPE_OBJ,
1879                         msg_size, &flags);
1880         KBASE_DEBUG_ASSERT(buffer);
1881
1882         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1883         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1884         pos = kbasep_tlstream_write_bytes(
1885                         buffer, pos, &atom, sizeof(atom));
1886         pos = kbasep_tlstream_write_bytes(
1887                         buffer, pos, &as, sizeof(as));
1888         KBASE_DEBUG_ASSERT(msg_size == pos);
1889
1890         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1891 }
1892
1893 void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as)
1894 {
1895         const u32     msg_id = KBASE_TL_NRET_ATOM_AS;
1896         const size_t  msg_size =
1897                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as);
1898         unsigned long flags;
1899         char          *buffer;
1900         size_t        pos = 0;
1901
1902         buffer = kbasep_tlstream_msgbuf_acquire(
1903                         TL_STREAM_TYPE_OBJ,
1904                         msg_size, &flags);
1905         KBASE_DEBUG_ASSERT(buffer);
1906
1907         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1908         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1909         pos = kbasep_tlstream_write_bytes(
1910                         buffer, pos, &atom, sizeof(atom));
1911         pos = kbasep_tlstream_write_bytes(
1912                         buffer, pos, &as, sizeof(as));
1913         KBASE_DEBUG_ASSERT(msg_size == pos);
1914
1915         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1916 }
1917
1918 void __kbase_tlstream_tl_attrib_atom_config(
1919                 void *atom, u64 jd, u64 affinity, u32 config)
1920 {
1921         const u32     msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG;
1922         const size_t  msg_size =
1923                 sizeof(msg_id) + sizeof(u64) + sizeof(atom) +
1924                 sizeof(jd) + sizeof(affinity) + sizeof(config);
1925         unsigned long flags;
1926         char          *buffer;
1927         size_t        pos = 0;
1928
1929         buffer = kbasep_tlstream_msgbuf_acquire(
1930                         TL_STREAM_TYPE_OBJ,
1931                         msg_size, &flags);
1932         KBASE_DEBUG_ASSERT(buffer);
1933
1934         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1935         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1936         pos = kbasep_tlstream_write_bytes(
1937                         buffer, pos, &atom, sizeof(atom));
1938         pos = kbasep_tlstream_write_bytes(
1939                         buffer, pos, &jd, sizeof(jd));
1940         pos = kbasep_tlstream_write_bytes(
1941                         buffer, pos, &affinity, sizeof(affinity));
1942         pos = kbasep_tlstream_write_bytes(
1943                         buffer, pos, &config, sizeof(config));
1944         KBASE_DEBUG_ASSERT(msg_size == pos);
1945
1946         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1947 }
1948
1949 void __kbase_tlstream_tl_attrib_as_config(
1950                 void *as, u64 transtab, u64 memattr, u64 transcfg)
1951 {
1952         const u32     msg_id = KBASE_TL_ATTRIB_AS_CONFIG;
1953         const size_t  msg_size =
1954                 sizeof(msg_id) + sizeof(u64) + sizeof(as) +
1955                 sizeof(transtab) + sizeof(memattr) + sizeof(transcfg);
1956         unsigned long flags;
1957         char          *buffer;
1958         size_t        pos = 0;
1959
1960         buffer = kbasep_tlstream_msgbuf_acquire(
1961                         TL_STREAM_TYPE_OBJ,
1962                         msg_size, &flags);
1963         KBASE_DEBUG_ASSERT(buffer);
1964
1965         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1966         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1967         pos = kbasep_tlstream_write_bytes(
1968                         buffer, pos, &as, sizeof(as));
1969         pos = kbasep_tlstream_write_bytes(
1970                         buffer, pos, &transtab, sizeof(transtab));
1971         pos = kbasep_tlstream_write_bytes(
1972                         buffer, pos, &memattr, sizeof(memattr));
1973         pos = kbasep_tlstream_write_bytes(
1974                         buffer, pos, &transcfg, sizeof(transcfg));
1975         KBASE_DEBUG_ASSERT(msg_size == pos);
1976
1977         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
1978 }
1979
1980 void __kbase_tlstream_jd_gpu_soft_reset(void *gpu)
1981 {
1982         const u32     msg_id = KBASE_JD_GPU_SOFT_RESET;
1983         const size_t  msg_size =
1984                 sizeof(msg_id) + sizeof(u64) + sizeof(gpu);
1985         unsigned long flags;
1986         char          *buffer;
1987         size_t        pos = 0;
1988
1989         buffer = kbasep_tlstream_msgbuf_acquire(
1990                         TL_STREAM_TYPE_OBJ,
1991                         msg_size, &flags);
1992         KBASE_DEBUG_ASSERT(buffer);
1993
1994         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
1995         pos = kbasep_tlstream_write_timestamp(buffer, pos);
1996         pos = kbasep_tlstream_write_bytes(
1997                         buffer, pos, &gpu, sizeof(gpu));
1998         KBASE_DEBUG_ASSERT(msg_size == pos);
1999
2000         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
2001 }
2002
2003 /*****************************************************************************/
2004
2005 void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state)
2006 {
2007         const u32     msg_id = KBASE_AUX_PM_STATE;
2008         const size_t  msg_size =
2009                 sizeof(msg_id) + sizeof(u64) + sizeof(core_type) +
2010                 sizeof(state);
2011         unsigned long flags;
2012         char          *buffer;
2013         size_t        pos = 0;
2014
2015         buffer = kbasep_tlstream_msgbuf_acquire(
2016                         TL_STREAM_TYPE_AUX,
2017                         msg_size, &flags);
2018         KBASE_DEBUG_ASSERT(buffer);
2019
2020         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2021         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2022         pos = kbasep_tlstream_write_bytes(
2023                         buffer, pos, &core_type, sizeof(core_type));
2024         pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state));
2025         KBASE_DEBUG_ASSERT(msg_size == pos);
2026
2027         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2028 }
2029
2030 void __kbase_tlstream_aux_issue_job_softstop(void *katom)
2031 {
2032         const u32     msg_id = KBASE_AUX_ISSUE_JOB_SOFTSTOP;
2033         const size_t  msg_size =
2034                 sizeof(msg_id) + sizeof(u64) + sizeof(katom);
2035         unsigned long flags;
2036         char          *buffer;
2037         size_t        pos = 0;
2038
2039         buffer = kbasep_tlstream_msgbuf_acquire(
2040                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2041         KBASE_DEBUG_ASSERT(buffer);
2042
2043         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2044         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2045         pos = kbasep_tlstream_write_bytes(buffer, pos, &katom, sizeof(katom));
2046         KBASE_DEBUG_ASSERT(msg_size == pos);
2047
2048         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2049 }
2050
2051 void __kbase_tlstream_aux_job_softstop(u32 js_id)
2052 {
2053         const u32     msg_id = KBASE_AUX_JOB_SOFTSTOP;
2054         const size_t  msg_size =
2055                 sizeof(msg_id) + sizeof(u64) + sizeof(js_id);
2056         unsigned long flags;
2057         char          *buffer;
2058         size_t        pos = 0;
2059
2060         buffer = kbasep_tlstream_msgbuf_acquire(
2061                         TL_STREAM_TYPE_AUX,
2062                         msg_size, &flags);
2063         KBASE_DEBUG_ASSERT(buffer);
2064
2065         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2066         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2067         pos = kbasep_tlstream_write_bytes(buffer, pos, &js_id, sizeof(js_id));
2068         KBASE_DEBUG_ASSERT(msg_size == pos);
2069
2070         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2071 }
2072
2073 /**
2074  * __kbase_tlstream_aux_job_softstop_ex_record - record the trace point
2075  * @katom: the atom that has been soft-stopped
2076  * @job_type: the job type
2077  */
2078 static void __kbase_tlstream_aux_job_softstop_ex_record(
2079                 void *katom, u32 job_type)
2080 {
2081         const u32     msg_id = KBASE_AUX_JOB_SOFTSTOP_EX;
2082         const size_t  msg_size =
2083                 sizeof(msg_id) + sizeof(u64) + sizeof(katom) + sizeof(job_type);
2084         unsigned long flags;
2085         char          *buffer;
2086         size_t        pos = 0;
2087
2088         buffer = kbasep_tlstream_msgbuf_acquire(
2089                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2090         KBASE_DEBUG_ASSERT(buffer);
2091
2092         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2093         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2094         pos = kbasep_tlstream_write_bytes(buffer, pos, &katom, sizeof(katom));
2095         pos = kbasep_tlstream_write_bytes(
2096                         buffer, pos, &job_type, sizeof(job_type));
2097         KBASE_DEBUG_ASSERT(msg_size == pos);
2098
2099         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2100 }
2101
2102 void __kbase_tlstream_aux_job_softstop_ex(struct kbase_jd_atom *katom)
2103 {
2104         struct kbase_context *kctx = katom->kctx;
2105         u64 jd = katom->jc;
2106
2107         while (jd != 0) {
2108                 struct job_descriptor_header *job;
2109                 struct kbase_vmap_struct map;
2110
2111                 job = kbase_vmap(kctx, jd, sizeof(*job), &map);
2112                 if (!job) {
2113                         dev_err(kctx->kbdev->dev,
2114                                 "__kbase_tlstream_aux_job_softstop_ex: failed to map job descriptor 0x%llx for atom 0x%p\n",
2115                                 jd, (void *)katom);
2116                         break;
2117                 }
2118                 if (job->exception_status != BASE_JD_EVENT_STOPPED) {
2119                         kbase_vunmap(kctx, &map);
2120                         break;
2121                 }
2122
2123                 __kbase_tlstream_aux_job_softstop_ex_record(
2124                                 katom, job->job_type);
2125
2126                 jd = job->job_descriptor_size ?
2127                         job->next_job._64 : job->next_job._32;
2128                 kbase_vunmap(kctx, &map);
2129         }
2130 }
2131
2132 void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change)
2133 {
2134         const u32     msg_id = KBASE_AUX_PAGEFAULT;
2135         const size_t  msg_size =
2136                 sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) +
2137                 sizeof(page_count_change);
2138         unsigned long flags;
2139         char          *buffer;
2140         size_t        pos = 0;
2141
2142         buffer = kbasep_tlstream_msgbuf_acquire(
2143                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2144         KBASE_DEBUG_ASSERT(buffer);
2145
2146         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2147         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2148         pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr));
2149         pos = kbasep_tlstream_write_bytes(
2150                         buffer, pos,
2151                         &page_count_change, sizeof(page_count_change));
2152         KBASE_DEBUG_ASSERT(msg_size == pos);
2153
2154         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2155 }
2156
2157 void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count)
2158 {
2159         const u32     msg_id = KBASE_AUX_PAGESALLOC;
2160         const size_t  msg_size =
2161                 sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) +
2162                 sizeof(page_count);
2163         unsigned long flags;
2164         char          *buffer;
2165         size_t        pos = 0;
2166
2167         buffer = kbasep_tlstream_msgbuf_acquire(
2168                         TL_STREAM_TYPE_AUX, msg_size, &flags);
2169         KBASE_DEBUG_ASSERT(buffer);
2170
2171         pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
2172         pos = kbasep_tlstream_write_timestamp(buffer, pos);
2173         pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr));
2174         pos = kbasep_tlstream_write_bytes(
2175                         buffer, pos, &page_count, sizeof(page_count));
2176         KBASE_DEBUG_ASSERT(msg_size == pos);
2177
2178         kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
2179 }
2180