drivers/gpu/arm/midgard/mali_kbase_replay.c

   1 /*
   2  *
   3  * (C) COPYRIGHT ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /**
  19  * @file mali_kbase_replay.c
  20  * Replay soft job handlers
  21  */
  22
  23 #include <mali_kbase_config.h>
  24 #include <mali_kbase.h>
  25 #include <mali_kbase_mem.h>
  26 #include <mali_kbase_debug.h>
  27
  28 #define JOB_NOT_STARTED 0
  29
  30 #define JOB_TYPE_MASK      0xfe
  31 #define JOB_TYPE_NULL      (1 << 1)
  32 #define JOB_TYPE_VERTEX    (5 << 1)
  33 #define JOB_TYPE_TILER     (7 << 1)
  34 #define JOB_TYPE_FUSED     (8 << 1)
  35 #define JOB_TYPE_FRAGMENT  (9 << 1)
  36
  37 #define JOB_FLAG_DESC_SIZE           (1 << 0)
  38 #define JOB_FLAG_PERFORM_JOB_BARRIER (1 << 8)
  39
  40 #define JOB_HEADER_32_FBD_OFFSET (31*4)
  41
  42 #define FBD_POINTER_MASK (~0x3f)
  43
  44 #define SFBD_TILER_OFFSET (48*4)
  45
  46 #define MFBD_TILER_FLAGS_OFFSET (15*4)
  47 #define MFBD_TILER_OFFSET       (16*4)
  48
  49 #define FBD_HIERARCHY_WEIGHTS 8
  50 #define FBD_HIERARCHY_MASK_MASK 0x1fff
  51
  52 #define FBD_TYPE 1
  53
  54 #define HIERARCHY_WEIGHTS 13
  55
  56 #define JOB_HEADER_ID_MAX                 0xffff
  57
  58 typedef struct job_head
  59 {
  60         u32 status;
  61         u32 not_complete_index;
  62         u64 fault_addr;
  63         u16 flags;
  64         u16 index;
  65         u16 dependencies[2];
  66         union
  67         {
  68                 u64 _64;
  69                 u32 _32;
  70         } next;
  71         u32 x[2];
  72         union
  73         {
  74                 u64 _64;
  75                 u32 _32;
  76         } fragment_fbd;
  77 } job_head;
  78
  79 static void dump_job_head(kbase_context *kctx, char *head_str, job_head *job)
  80 {
  81 #ifdef CONFIG_MALI_DEBUG
  82         struct device *dev = kctx->kbdev->dev;
  83
  84         KBASE_LOG(2, dev, "%s\n", head_str);
  85         KBASE_LOG(2, dev, "addr               = %p\n"
  86                                         "status             = %x\n"
  87                                         "not_complete_index = %x\n"
  88                                         "fault_addr         = %llx\n"
  89                                         "flags              = %x\n"
  90                                         "index              = %x\n"
  91                                         "dependencies       = %x,%x\n",
  92                                                                            job,
  93                                                                    job->status,
  94                                                        job->not_complete_index,
  95                                                                job->fault_addr,
  96                                                                     job->flags,
  97                                                                     job->index,
  98                                                           job->dependencies[0],
  99                                                          job->dependencies[1]);
 100
 101         if (job->flags & JOB_FLAG_DESC_SIZE)
 102                 KBASE_LOG(2, dev, "next               = %llx\n", job->next._64);
 103         else
 104                 KBASE_LOG(2, dev, "next               = %x\n", job->next._32);
 105 #endif
 106 }
 107
 108
 109 static void *kbasep_map_page(kbase_context *kctx, mali_addr64 gpu_addr,
 110                                                                 u64 *phys_addr)
 111 {
 112         void *cpu_addr = NULL;
 113         u64 page_index;
 114         kbase_va_region *region;
 115         phys_addr_t *page_array;
 116
 117         region = kbase_region_tracker_find_region_enclosing_address(kctx,
 118                                                                      gpu_addr);
 119         if (!region || (region->flags & KBASE_REG_FREE))
 120                 return NULL;
 121
 122         page_index = (gpu_addr >> PAGE_SHIFT) - region->start_pfn;
 123         if (page_index >= kbase_reg_current_backed_size(region))
 124                 return NULL;
 125
 126         page_array = kbase_get_phy_pages(region);
 127         if (!page_array)
 128                 return NULL;
 129
 130         cpu_addr = kmap_atomic(pfn_to_page(PFN_DOWN(page_array[page_index])));
 131         if (!cpu_addr)
 132                 return NULL;
 133
 134         if (phys_addr)
 135                 *phys_addr = page_array[page_index];
 136
 137         return cpu_addr + (gpu_addr & ~PAGE_MASK);
 138 }
 139
 140 static void *kbasep_map_page_sync(kbase_context *kctx, mali_addr64 gpu_addr,
 141                                                                 u64 *phys_addr)
 142 {
 143         void *cpu_addr = kbasep_map_page(kctx, gpu_addr, phys_addr);
 144
 145         if (!cpu_addr)
 146                 return NULL;
 147
 148         kbase_sync_to_cpu(*phys_addr,
 149                                  (void *)((uintptr_t)cpu_addr & PAGE_MASK),
 150                                                                     PAGE_SIZE);
 151
 152         return cpu_addr;
 153 }
 154
 155 static void kbasep_unmap_page(void *cpu_addr)
 156 {
 157         kunmap_atomic((void *)((uintptr_t)cpu_addr & PAGE_MASK));
 158 }
 159
 160 static void kbasep_unmap_page_sync(void *cpu_addr, u64 phys_addr)
 161 {
 162         kbase_sync_to_memory(phys_addr,
 163                                  (void *)((uintptr_t)cpu_addr & PAGE_MASK),
 164                                                                     PAGE_SIZE);
 165
 166         kunmap_atomic((void *)((uintptr_t)cpu_addr & PAGE_MASK));
 167 }
 168
 169 static mali_error kbasep_replay_reset_sfbd(kbase_context *kctx,
 170                                            mali_addr64 fbd_address,
 171                                            mali_addr64 tiler_heap_free,
 172                                            u16 hierarchy_mask,
 173                                            u32 default_weight)
 174 {
 175         u64 phys_addr;
 176         struct
 177         {
 178                 u32 padding_1[1];
 179                 u32 flags;
 180                 u64 padding_2[2];
 181                 u64 heap_free_address;
 182                 u32 padding[8];
 183                 u32 weights[FBD_HIERARCHY_WEIGHTS];
 184         } *fbd_tiler;
 185         struct device *dev = kctx->kbdev->dev;
 186
 187         KBASE_LOG(2, dev, "fbd_address: %llx\n", fbd_address);
 188
 189         fbd_tiler = kbasep_map_page_sync(kctx, fbd_address + SFBD_TILER_OFFSET,
 190                                                                    &phys_addr);
 191         if (!fbd_tiler) {
 192                 dev_err(dev, "kbasep_replay_reset_fbd: failed to map fbd\n");
 193                 return MALI_ERROR_FUNCTION_FAILED;
 194         }
 195 #ifdef CONFIG_MALI_DEBUG
 196         KBASE_LOG(2, dev, "FBD tiler:\n"
 197                                 "flags = %x\n"
 198                                 "heap_free_address = %llx\n",
 199                                                               fbd_tiler->flags,
 200                                                  fbd_tiler->heap_free_address);
 201 #endif
 202         if (hierarchy_mask) {
 203                 u32 weights[HIERARCHY_WEIGHTS];
 204                 u16 old_hierarchy_mask = fbd_tiler->flags &
 205                                                        FBD_HIERARCHY_MASK_MASK;
 206                 int i, j = 0;
 207
 208                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 209                         if (old_hierarchy_mask & (1 << i)) {
 210                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 211                                 weights[i] = fbd_tiler->weights[j++];
 212                         } else {
 213                                 weights[i] = default_weight;
 214                         }
 215                 }
 216
 217
 218                 KBASE_LOG(2, dev,
 219                               "Old hierarchy mask=%x  New hierarchy mask=%x\n",
 220                                            old_hierarchy_mask, hierarchy_mask);
 221                 for (i = 0; i < HIERARCHY_WEIGHTS; i++)
 222                         KBASE_LOG(2, dev, " Hierarchy weight %02d: %08x\n",
 223                                                                 i, weights[i]);
 224
 225                 j = 0;
 226
 227                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 228                         if (hierarchy_mask & (1 << i)) {
 229                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 230
 231                                 KBASE_LOG(2, dev,
 232                                 " Writing hierarchy level %02d (%08x) to %d\n",
 233                                                              i, weights[i], j);
 234
 235                                 fbd_tiler->weights[j++] = weights[i];
 236                         }
 237                 }
 238
 239                 for (; j < FBD_HIERARCHY_WEIGHTS; j++)
 240                         fbd_tiler->weights[j] = 0;
 241
 242                 fbd_tiler->flags = hierarchy_mask | (1 << 16);
 243         }
 244
 245         fbd_tiler->heap_free_address = tiler_heap_free;
 246
 247         KBASE_LOG(2, dev, "heap_free_address=%llx flags=%x\n",
 248                                fbd_tiler->heap_free_address, fbd_tiler->flags);
 249
 250         kbasep_unmap_page_sync(fbd_tiler, phys_addr);
 251
 252         return MALI_ERROR_NONE;
 253 }
 254
 255 static mali_error kbasep_replay_reset_mfbd(kbase_context *kctx,
 256                                            mali_addr64 fbd_address,
 257                                            mali_addr64 tiler_heap_free,
 258                                            u16 hierarchy_mask,
 259                                            u32 default_weight)
 260 {
 261         u64 phys_addr, phys_addr_flags;
 262         struct
 263         {
 264                 u64 padding_1[2];
 265                 u64 heap_free_address;
 266                 u64 padding_2;
 267                 u32 weights[FBD_HIERARCHY_WEIGHTS];
 268         } *fbd_tiler;
 269         u32 *fbd_tiler_flags;
 270         mali_bool flags_different_page;
 271         struct device *dev = kctx->kbdev->dev;
 272
 273         KBASE_LOG(2, dev, "fbd_address: %llx\n", fbd_address);
 274
 275         fbd_tiler = kbasep_map_page_sync(kctx, fbd_address + MFBD_TILER_OFFSET,
 276                                                                    &phys_addr);
 277         if (((fbd_address + MFBD_TILER_OFFSET) & PAGE_MASK) !=
 278             ((fbd_address + MFBD_TILER_FLAGS_OFFSET) & PAGE_MASK)) {
 279                 flags_different_page = MALI_TRUE;
 280                 fbd_tiler_flags = kbasep_map_page_sync(kctx,
 281                                          fbd_address + MFBD_TILER_FLAGS_OFFSET,
 282                                                              &phys_addr_flags);
 283         } else {
 284                 flags_different_page = MALI_FALSE;
 285                 fbd_tiler_flags = (u32 *)((uintptr_t)fbd_tiler -
 286                                   MFBD_TILER_OFFSET + MFBD_TILER_FLAGS_OFFSET);
 287         }
 288
 289         if (!fbd_tiler || !fbd_tiler_flags) {
 290                 dev_err(dev, "kbasep_replay_reset_fbd: failed to map fbd\n");
 291
 292                 if (fbd_tiler_flags && flags_different_page)
 293                         kbasep_unmap_page_sync(fbd_tiler_flags,
 294                                                               phys_addr_flags);
 295                 if (fbd_tiler)
 296                         kbasep_unmap_page_sync(fbd_tiler, phys_addr);
 297
 298                 return MALI_ERROR_FUNCTION_FAILED;
 299         }
 300 #ifdef CONFIG_MALI_DEBUG
 301         KBASE_LOG(2, dev, "FBD tiler:\n"
 302                                 "heap_free_address = %llx\n",
 303                                  fbd_tiler->heap_free_address);
 304 #endif
 305         if (hierarchy_mask) {
 306                 u32 weights[HIERARCHY_WEIGHTS];
 307                 u16 old_hierarchy_mask = (*fbd_tiler_flags) &
 308                                                        FBD_HIERARCHY_MASK_MASK;
 309                 int i, j = 0;
 310
 311                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 312                         if (old_hierarchy_mask & (1 << i)) {
 313                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 314                                 weights[i] = fbd_tiler->weights[j++];
 315                         }
 316                         else
 317                                 weights[i] = default_weight;
 318                 }
 319
 320
 321                 KBASE_LOG(2, dev,
 322                               "Old hierarchy mask=%x  New hierarchy mask=%x\n",
 323                                            old_hierarchy_mask, hierarchy_mask);
 324                 for (i = 0; i < HIERARCHY_WEIGHTS; i++)
 325                         KBASE_LOG(2, dev, " Hierarchy weight %02d: %08x\n",
 326                                                                 i, weights[i]);
 327
 328                 j = 0;
 329
 330                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 331                         if (hierarchy_mask & (1 << i)) {
 332                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 333
 334                                 KBASE_LOG(2, dev,
 335                                 " Writing hierarchy level %02d (%08x) to %d\n",
 336                                                              i, weights[i], j);
 337
 338                                 fbd_tiler->weights[j++] = weights[i];
 339                         }
 340                 }
 341
 342                 for (; j < FBD_HIERARCHY_WEIGHTS; j++)
 343                         fbd_tiler->weights[j] = 0;
 344
 345                 *fbd_tiler_flags = hierarchy_mask | (1 << 16);
 346         }
 347
 348         fbd_tiler->heap_free_address = tiler_heap_free;
 349
 350         if (flags_different_page)
 351                 kbasep_unmap_page_sync(fbd_tiler_flags, phys_addr_flags);
 352
 353         kbasep_unmap_page_sync(fbd_tiler, phys_addr);
 354
 355         return MALI_ERROR_NONE;
 356 }
 357
 358 /**
 359  * @brief Reset the status of an FBD pointed to by a tiler job
 360  *
 361  * This performs two functions :
 362  * - Set the hierarchy mask
 363  * - Reset the tiler free heap address
 364  *
 365  * @param[in] kctx              Context pointer
 366  * @param[in] job_header        Address of job header to reset.
 367  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 368  * @param[in] hierarchy_mask    The hierarchy mask to use
 369  * @param[in] default_weight    Default hierarchy weight to write when no other
 370  *                              weight is given in the FBD
 371  * @param[in] job_64            MALI_TRUE if this job is using 64-bit
 372  *                              descriptors
 373  *
 374  * @return MALI_ERROR_NONE on success, error code on failure
 375  */
 376 static mali_error kbasep_replay_reset_tiler_job(kbase_context *kctx,
 377                                                 mali_addr64 job_header,
 378                                                 mali_addr64 tiler_heap_free,
 379                                                 u16 hierarchy_mask,
 380                                                 u32 default_weight,
 381                                                 mali_bool job_64)
 382 {
 383         mali_addr64 fbd_address;
 384
 385         if (job_64) {
 386                 dev_err(kctx->kbdev->dev,
 387                                       "64-bit job descriptor not supported\n");
 388                 return MALI_ERROR_FUNCTION_FAILED;
 389         } else {
 390                 u32 *job_ext;
 391
 392                 job_ext = kbasep_map_page(kctx,
 393                                          job_header + JOB_HEADER_32_FBD_OFFSET,
 394                                                                          NULL);
 395                 if (!job_ext) {
 396                         dev_err(kctx->kbdev->dev,
 397                           "kbasep_replay_reset_tiler_job: failed to map jc\n");
 398                         return MALI_ERROR_FUNCTION_FAILED;
 399                 }
 400
 401                 fbd_address = *job_ext;
 402
 403                 kbasep_unmap_page(job_ext);
 404         }
 405
 406         if (fbd_address & FBD_TYPE) {
 407                 return kbasep_replay_reset_mfbd(kctx,
 408                                                 fbd_address & FBD_POINTER_MASK,
 409                                                 tiler_heap_free,
 410                                                 hierarchy_mask,
 411                                                 default_weight);
 412         } else {
 413                 return kbasep_replay_reset_sfbd(kctx,
 414                                                 fbd_address & FBD_POINTER_MASK,
 415                                                 tiler_heap_free,
 416                                                 hierarchy_mask,
 417                                                 default_weight);
 418         }
 419 }
 420
 421 /**
 422  * @brief Reset the status of a job
 423  *
 424  * This performs the following functions :
 425  *
 426  * - Reset the Job Status field of each job to NOT_STARTED.
 427  * - Set the Job Type field of any Vertex Jobs to Null Job.
 428  * - For any jobs using an FBD, set the Tiler Heap Free field to the value of
 429  *   the tiler_heap_free parameter, and set the hierarchy level mask to the
 430  *   hier_mask parameter.
 431  * - Offset HW dependencies by the hw_job_id_offset parameter
 432  * - Set the Perform Job Barrier flag if this job is the first in the chain
 433  * - Read the address of the next job header
 434  *
 435  * @param[in] kctx              Context pointer
 436  * @param[in,out] job_header    Address of job header to reset. Set to address
 437  *                              of next job header on exit.
 438  * @param[in] prev_jc           Previous job chain to link to, if this job is
 439  *                              the last in the chain.
 440  * @param[in] hw_job_id_offset  Offset for HW job IDs
 441  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 442  * @param[in] hierarchy_mask    The hierarchy mask to use
 443  * @param[in] default_weight    Default hierarchy weight to write when no other
 444  *                              weight is given in the FBD
 445  * @param[in] first_in_chain    MALI_TRUE if this job is the first in the chain
 446  * @param[in] fragment_chain    MALI_TRUE if this job is in the fragment chain
 447  *
 448  * @return MALI_ERROR_NONE on success, error code on failure
 449  */
 450 static mali_error kbasep_replay_reset_job(kbase_context *kctx,
 451                                                 mali_addr64 *job_header,
 452                                                 mali_addr64 prev_jc,
 453                                                 mali_addr64 tiler_heap_free,
 454                                                 u16 hierarchy_mask,
 455                                                 u32 default_weight,
 456                                                 u16 hw_job_id_offset,
 457                                                 mali_bool first_in_chain,
 458                                                 mali_bool fragment_chain)
 459 {
 460         job_head *job;
 461         u64 phys_addr;
 462         mali_addr64 new_job_header;
 463         struct device *dev = kctx->kbdev->dev;
 464
 465         job = kbasep_map_page_sync(kctx, *job_header, &phys_addr);
 466         if (!job) {
 467                 dev_err(dev, "kbasep_replay_parse_jc: failed to map jc\n");
 468                 return MALI_ERROR_FUNCTION_FAILED;
 469         }
 470
 471         dump_job_head(kctx, "Job header:", job);
 472
 473         if (job->status == JOB_NOT_STARTED && !fragment_chain) {
 474                 dev_err(dev, "Job already not started\n");
 475                 kbasep_unmap_page_sync(job, phys_addr);
 476                 return MALI_ERROR_FUNCTION_FAILED;
 477         }
 478         job->status = JOB_NOT_STARTED;
 479
 480         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_VERTEX)
 481                 job->flags = (job->flags & ~JOB_TYPE_MASK) | JOB_TYPE_NULL;
 482
 483         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_FUSED) {
 484                 dev_err(dev, "Fused jobs can not be replayed\n");
 485                 kbasep_unmap_page_sync(job, phys_addr);
 486                 return MALI_ERROR_FUNCTION_FAILED;
 487         }
 488
 489         if (first_in_chain)
 490                 job->flags |= JOB_FLAG_PERFORM_JOB_BARRIER;
 491
 492         if ((job->dependencies[0] + hw_job_id_offset) > JOB_HEADER_ID_MAX ||
 493             (job->dependencies[1] + hw_job_id_offset) > JOB_HEADER_ID_MAX ||
 494             (job->index + hw_job_id_offset) > JOB_HEADER_ID_MAX) {
 495                 dev_err(dev, "Job indicies/dependencies out of valid range\n");
 496                 kbasep_unmap_page_sync(job, phys_addr);
 497                 return MALI_ERROR_FUNCTION_FAILED;
 498         }
 499
 500         if (job->dependencies[0])
 501                 job->dependencies[0] += hw_job_id_offset;
 502         if (job->dependencies[1])
 503                 job->dependencies[1] += hw_job_id_offset;
 504
 505         job->index += hw_job_id_offset;
 506
 507         if (job->flags & JOB_FLAG_DESC_SIZE) {
 508                 new_job_header = job->next._64;
 509                 if (!job->next._64)
 510                         job->next._64 = prev_jc;
 511         } else {
 512                 new_job_header = job->next._32;
 513                 if (!job->next._32)
 514                         job->next._32 = prev_jc;
 515         }
 516         dump_job_head(kctx, "Updated to:", job);
 517
 518         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_TILER) {
 519                 kbasep_unmap_page_sync(job, phys_addr);
 520                 if (kbasep_replay_reset_tiler_job(kctx, *job_header,
 521                                         tiler_heap_free, hierarchy_mask,
 522                                         default_weight,
 523                                         job->flags & JOB_FLAG_DESC_SIZE) !=
 524                                                         MALI_ERROR_NONE)
 525                         return MALI_ERROR_FUNCTION_FAILED;
 526
 527         } else if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_FRAGMENT) {
 528                 u64 fbd_address;
 529
 530                 if (job->flags & JOB_FLAG_DESC_SIZE) {
 531                         kbasep_unmap_page_sync(job, phys_addr);
 532                         dev_err(dev, "64-bit job descriptor not supported\n");
 533                         return MALI_ERROR_FUNCTION_FAILED;
 534                 } else {
 535                         fbd_address = (u64)job->fragment_fbd._32;
 536                 }
 537
 538                 kbasep_unmap_page_sync(job, phys_addr);
 539
 540                 if (fbd_address & FBD_TYPE) {
 541                         if (kbasep_replay_reset_mfbd(kctx,
 542                                                 fbd_address & FBD_POINTER_MASK,
 543                                                 tiler_heap_free,
 544                                                 hierarchy_mask,
 545                                                 default_weight) !=
 546                                                                MALI_ERROR_NONE)
 547                                 return MALI_ERROR_FUNCTION_FAILED;
 548                 } else {
 549                         if (kbasep_replay_reset_sfbd(kctx,
 550                                                 fbd_address & FBD_POINTER_MASK,
 551                                                 tiler_heap_free,
 552                                                 hierarchy_mask,
 553                                                 default_weight) !=
 554                                                                MALI_ERROR_NONE)
 555                                 return MALI_ERROR_FUNCTION_FAILED;
 556                 }
 557         } else {
 558                 kbasep_unmap_page_sync(job, phys_addr);
 559         }
 560
 561         *job_header = new_job_header;
 562
 563         return MALI_ERROR_NONE;
 564 }
 565
 566 /**
 567  * @brief Find the highest job ID in a job chain
 568  *
 569  * @param[in] kctx        Context pointer
 570  * @param[in] jc          Job chain start address
 571  * @param[out] hw_job_id  Highest job ID in chain
 572  *
 573  * @return MALI_ERROR_NONE on success, error code on failure
 574  */
 575 static mali_error kbasep_replay_find_hw_job_id(kbase_context *kctx,
 576                                                 mali_addr64 jc,
 577                                                 u16 *hw_job_id)
 578 {
 579         while (jc) {
 580                 job_head *job;
 581                 u64 phys_addr;
 582
 583                 KBASE_LOG(2, kctx->kbdev->dev,
 584                         "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc);
 585
 586                 job = kbasep_map_page_sync(kctx, jc, &phys_addr);
 587                 if (!job) {
 588                         dev_err(kctx->kbdev->dev, "failed to map jc\n");
 589
 590                         return MALI_ERROR_FUNCTION_FAILED;
 591                 }
 592
 593                 if (job->index > *hw_job_id)
 594                         *hw_job_id = job->index;
 595
 596                 if (job->flags & JOB_FLAG_DESC_SIZE)
 597                         jc = job->next._64;
 598                 else
 599                         jc = job->next._32;
 600
 601                 kbasep_unmap_page_sync(job, phys_addr);
 602         }
 603
 604         return MALI_ERROR_NONE;
 605 }
 606
 607 /**
 608  * @brief Reset the status of a number of jobs
 609  *
 610  * This function walks the provided job chain, and calls
 611  * kbasep_replay_reset_job for each job. It also links the job chain to the
 612  * provided previous job chain.
 613  *
 614  * The function will fail if any of the jobs passed already have status of
 615  * NOT_STARTED.
 616  *
 617  * @param[in] kctx              Context pointer
 618  * @param[in] jc                Job chain to be processed
 619  * @param[in] prev_jc           Job chain to be added to. May be NULL
 620  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 621  * @param[in] hierarchy_mask    The hierarchy mask to use
 622  * @param[in] default_weight    Default hierarchy weight to write when no other
 623  *                              weight is given in the FBD
 624  * @param[in] hw_job_id_offset  Offset for HW job IDs
 625  * @param[in] fragment_chain    MAIL_TRUE if this chain is the fragment chain
 626  *
 627  * @return MALI_ERROR_NONE on success, error code otherwise
 628  */
 629 static mali_error kbasep_replay_parse_jc(kbase_context *kctx,
 630                                                 mali_addr64 jc,
 631                                                 mali_addr64 prev_jc,
 632                                                 mali_addr64 tiler_heap_free,
 633                                                 u16 hierarchy_mask,
 634                                                 u32 default_weight,
 635                                                 u16 hw_job_id_offset,
 636                                                 mali_bool fragment_chain)
 637 {
 638         mali_bool first_in_chain = MALI_TRUE;
 639         int nr_jobs = 0;
 640
 641         KBASE_LOG(2, kctx->kbdev->dev,
 642                               "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n",
 643                                                          jc, hw_job_id_offset);
 644
 645         while (jc) {
 646                 KBASE_LOG(2, kctx->kbdev->dev,
 647                                    "kbasep_replay_parse_jc: parsing jc=%llx\n",
 648                                                                            jc);
 649
 650                 if (kbasep_replay_reset_job(kctx, &jc, prev_jc,
 651                                 tiler_heap_free, hierarchy_mask,
 652                                 default_weight, hw_job_id_offset,
 653                                 first_in_chain, fragment_chain) !=
 654                                                              MALI_ERROR_NONE)
 655                         return MALI_ERROR_FUNCTION_FAILED;
 656
 657                 first_in_chain = MALI_FALSE;
 658
 659                 nr_jobs++;
 660                 if (fragment_chain &&
 661                                 nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) {
 662                         dev_err(kctx->kbdev->dev,
 663                                 "Exceeded maximum number of jobs in fragment chain\n");
 664                         return MALI_ERROR_FUNCTION_FAILED;
 665                 }
 666         }
 667
 668         return MALI_ERROR_NONE;
 669 }
 670
 671 /**
 672  * @brief Reset the status of a replay job, and set up dependencies
 673  *
 674  * This performs the actions to allow the replay job to be re-run following
 675  * completion of the passed dependency.
 676  *
 677  * @param[in] katom     The atom to be reset
 678  * @param[in] dep_atom  The dependency to be attached to the atom
 679  */
 680 static void kbasep_replay_reset_softjob(kbase_jd_atom *katom,
 681                                                        kbase_jd_atom *dep_atom)
 682 {
 683         katom->status = KBASE_JD_ATOM_STATE_QUEUED;
 684         katom->dep_atom[0] = dep_atom;
 685         list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]);
 686 }
 687
 688 /**
 689  * @brief Allocate an unused katom
 690  *
 691  * This will search the provided context for an unused katom, and will mark it
 692  * as KBASE_JD_ATOM_STATE_QUEUED.
 693  *
 694  * If no atoms are available then the function will fail.
 695  *
 696  * @param[in] kctx      Context pointer
 697  * @return An atom ID, or -1 on failure
 698  */
 699 static int kbasep_allocate_katom(kbase_context *kctx)
 700 {
 701         kbase_jd_context *jctx = &kctx->jctx;
 702         int i;
 703
 704         for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) {
 705                 if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) {
 706                         jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED;
 707                         KBASE_LOG(2, kctx->kbdev->dev,
 708                                   "kbasep_allocate_katom: Allocated atom %d\n",
 709                                                                             i);
 710                         return i;
 711                 }
 712         }
 713
 714         return -1;
 715 }
 716
 717 /**
 718  * @brief Release a katom
 719  *
 720  * This will mark the provided atom as available, and remove any dependencies.
 721  *
 722  * For use on error path.
 723  *
 724  * @param[in] kctx      Context pointer
 725  * @param[in] atom_id   ID of atom to release
 726  */
 727 static void kbasep_release_katom(kbase_context *kctx, int atom_id)
 728 {
 729         kbase_jd_context *jctx = &kctx->jctx;
 730
 731         KBASE_LOG(2, kctx->kbdev->dev,
 732                                     "kbasep_release_katom: Released atom %d\n",
 733                                                                       atom_id);
 734
 735         while (!list_empty(&jctx->atoms[atom_id].dep_head[0]))
 736                 list_del(jctx->atoms[atom_id].dep_head[0].next);
 737         while (!list_empty(&jctx->atoms[atom_id].dep_head[1]))
 738                 list_del(jctx->atoms[atom_id].dep_head[1].next);
 739
 740         jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED;
 741 }
 742
 743 static void kbasep_replay_create_atom(kbase_context *kctx,
 744                                       base_jd_atom_v2 *atom,
 745                                       int atom_nr,
 746                                       int prio)
 747 {
 748         atom->nr_extres = 0;
 749         atom->extres_list.value = NULL;
 750         atom->device_nr = 0;
 751         /* Convert priority back from NICE range */
 752         atom->prio = ((prio << 16) / ((20 << 16) / 128)) - 128;
 753         atom->atom_number = atom_nr;
 754
 755         atom->pre_dep[0] = 0;
 756         atom->pre_dep[1] = 0;
 757
 758         atom->udata.blob[0] = 0;
 759         atom->udata.blob[1] = 0;
 760 }
 761
 762 /**
 763  * @brief Create two atoms for the purpose of replaying jobs
 764  *
 765  * Two atoms are allocated and created. The jc pointer is not set at this
 766  * stage. The second atom has a dependency on the first. The remaining fields
 767  * are set up as follows :
 768  *
 769  * - No external resources. Any required external resources will be held by the
 770  *   replay atom.
 771  * - device_nr is set to 0. This is not relevant as
 772  *   BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set.
 773  * - Priority is inherited from the replay job.
 774  *
 775  * @param[out] t_atom      Atom to use for tiler jobs
 776  * @param[out] f_atom      Atom to use for fragment jobs
 777  * @param[in]  prio        Priority of new atom (inherited from replay soft
 778  *                         job)
 779  * @return MALI_ERROR_NONE on success, error code on failure
 780  */
 781 static mali_error kbasep_replay_create_atoms(kbase_context *kctx,
 782                                              base_jd_atom_v2 *t_atom,
 783                                              base_jd_atom_v2 *f_atom,
 784                                              int prio)
 785 {
 786         int t_atom_nr, f_atom_nr;
 787
 788         t_atom_nr = kbasep_allocate_katom(kctx);
 789         if (t_atom_nr < 0) {
 790                 dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
 791                 return MALI_ERROR_FUNCTION_FAILED;
 792         }
 793
 794         f_atom_nr = kbasep_allocate_katom(kctx);
 795         if (f_atom_nr < 0) {
 796                 dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
 797                 kbasep_release_katom(kctx, t_atom_nr);
 798                 return MALI_ERROR_FUNCTION_FAILED;
 799         }
 800
 801         kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio);
 802         kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio);
 803
 804         f_atom->pre_dep[0] = t_atom_nr;
 805
 806         return MALI_ERROR_NONE;
 807 }
 808
 809 #ifdef CONFIG_MALI_DEBUG
 810 static void payload_dump(kbase_context *kctx, base_jd_replay_payload *payload)
 811 {
 812         mali_addr64 next;
 813
 814         KBASE_LOG(2, kctx->kbdev->dev, "Tiler jc list :\n");
 815         next = payload->tiler_jc_list;
 816
 817         while (next) {
 818                 base_jd_replay_jc *jc_struct = kbasep_map_page(kctx, next, NULL);
 819
 820                 if (!jc_struct)
 821                         return;
 822
 823                 KBASE_LOG(2, kctx->kbdev->dev,
 824                                           "* jc_struct=%p jc=%llx next=%llx\n",
 825                                                                      jc_struct,
 826                                                                  jc_struct->jc,
 827                                                               jc_struct->next);
 828                 next = jc_struct->next;
 829
 830                 kbasep_unmap_page(jc_struct);
 831         }
 832 }
 833 #endif
 834
 835 /**
 836  * @brief Parse a base_jd_replay_payload provided by userspace
 837  *
 838  * This will read the payload from userspace, and parse the job chains.
 839  *
 840  * @param[in] kctx         Context pointer
 841  * @param[in] replay_atom  Replay soft job atom
 842  * @param[in] t_atom       Atom to use for tiler jobs
 843  * @param[in] f_atom       Atom to use for fragment jobs
 844  * @return  MALI_ERROR_NONE on success, error code on failure
 845  */
 846 static mali_error kbasep_replay_parse_payload(kbase_context *kctx,
 847                                               kbase_jd_atom *replay_atom,
 848                                               base_jd_atom_v2 *t_atom,
 849                                               base_jd_atom_v2 *f_atom)
 850 {
 851         base_jd_replay_payload *payload;
 852         mali_addr64 next;
 853         mali_addr64 prev_jc = 0;
 854         u16 hw_job_id_offset = 0;
 855         mali_error ret = MALI_ERROR_FUNCTION_FAILED;
 856         u64 phys_addr;
 857         struct device *dev = kctx->kbdev->dev;
 858
 859         KBASE_LOG(2, dev,
 860                         "kbasep_replay_parse_payload: replay_atom->jc = %llx  "
 861                         "sizeof(payload) = %d\n",
 862                                              replay_atom->jc, sizeof(payload));
 863
 864         kbase_gpu_vm_lock(kctx);
 865
 866         payload = kbasep_map_page_sync(kctx, replay_atom->jc, &phys_addr);
 867
 868         if (!payload) {
 869                 kbase_gpu_vm_unlock(kctx);
 870                 dev_err(dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n");
 871                 return MALI_ERROR_FUNCTION_FAILED;
 872         }
 873
 874 #ifdef CONFIG_MALI_DEBUG
 875         KBASE_LOG(2, dev, "kbasep_replay_parse_payload: payload=%p\n", payload);
 876         KBASE_LOG(2, dev, "Payload structure:\n"
 877                                         "tiler_jc_list            = %llx\n"
 878                                         "fragment_jc              = %llx\n"
 879                                         "tiler_heap_free          = %llx\n"
 880                                         "fragment_hierarchy_mask  = %x\n"
 881                                         "tiler_hierarchy_mask     = %x\n"
 882                                         "hierarchy_default_weight = %x\n"
 883                                         "tiler_core_req           = %x\n"
 884                                         "fragment_core_req        = %x\n",
 885                                                         payload->tiler_jc_list,
 886                                                           payload->fragment_jc,
 887                                                       payload->tiler_heap_free,
 888                                               payload->fragment_hierarchy_mask,
 889                                                  payload->tiler_hierarchy_mask,
 890                                              payload->hierarchy_default_weight,
 891                                                        payload->tiler_core_req,
 892                                                    payload->fragment_core_req);
 893         payload_dump(kctx, payload);
 894 #endif
 895
 896         t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER;
 897         f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER;
 898
 899         /* Sanity check core requirements*/
 900         if ((t_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
 901                                ~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_T ||
 902             (f_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
 903                               ~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_FS ||
 904              t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES ||
 905              f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
 906                 dev_err(dev, "Invalid core requirements\n");
 907                 goto out;
 908         }
 909
 910         /* Process tiler job chains */
 911         next = payload->tiler_jc_list;
 912         if (!next) {
 913                 dev_err(dev, "Invalid tiler JC list\n");
 914                 goto out;
 915         }
 916
 917         while (next) {
 918                 base_jd_replay_jc *jc_struct = kbasep_map_page(kctx, next, NULL);
 919                 mali_addr64 jc;
 920
 921                 if (!jc_struct) {
 922                         dev_err(dev, "Failed to map jc struct\n");
 923                         goto out;
 924                 }
 925
 926                 jc = jc_struct->jc;
 927                 next = jc_struct->next;
 928                 if (next)
 929                         jc_struct->jc = 0;
 930
 931                 kbasep_unmap_page(jc_struct);
 932
 933                 if (jc) {
 934                         u16 max_hw_job_id = 0;
 935
 936                         if (kbasep_replay_find_hw_job_id(kctx, jc,
 937                                             &max_hw_job_id) != MALI_ERROR_NONE)
 938                                 goto out;
 939
 940                         if (kbasep_replay_parse_jc(kctx, jc, prev_jc,
 941                                              payload->tiler_heap_free,
 942                                              payload->tiler_hierarchy_mask,
 943                                              payload->hierarchy_default_weight,
 944                                              hw_job_id_offset, MALI_FALSE) !=
 945                                                              MALI_ERROR_NONE) {
 946                                 goto out;
 947                         }
 948
 949                         hw_job_id_offset += max_hw_job_id;
 950
 951                         prev_jc = jc;
 952                 }
 953         }
 954         t_atom->jc = prev_jc;
 955
 956         /* Process fragment job chain */
 957         f_atom->jc = payload->fragment_jc;
 958         if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0,
 959                                          payload->tiler_heap_free,
 960                                          payload->fragment_hierarchy_mask,
 961                                          payload->hierarchy_default_weight, 0,
 962                                                MALI_TRUE) != MALI_ERROR_NONE) {
 963                 goto out;
 964         }
 965
 966         if (!t_atom->jc || !f_atom->jc) {
 967                 dev_err(dev, "Invalid payload\n");
 968                 goto out;
 969         }
 970
 971         KBASE_LOG(2, dev, "t_atom->jc=%llx f_atom->jc=%llx\n",
 972                                                        t_atom->jc, f_atom->jc);
 973         ret = MALI_ERROR_NONE;
 974
 975 out:
 976         kbasep_unmap_page_sync(payload, phys_addr);
 977
 978         kbase_gpu_vm_unlock(kctx);
 979
 980         return ret;
 981 }
 982
 983 /**
 984  * @brief Process a replay job
 985  *
 986  * Called from kbase_process_soft_job.
 987  *
 988  * On exit, if the job has completed, katom->event_code will have been updated.
 989  * If the job has not completed, and is replaying jobs, then the atom status
 990  * will have been reset to KBASE_JD_ATOM_STATE_QUEUED.
 991  *
 992  * @param[in] katom  The atom to be processed
 993  * @return           MALI_REPLAY_STATUS_COMPLETE  if the atom has completed
 994  *                   MALI_REPLAY_STATUS_REPLAYING if the atom is replaying jobs
 995  *                   Set MALI_REPLAY_FLAG_JS_RESCHED if
 996  *                   kbasep_js_try_schedule_head_ctx required
 997  */
 998 int kbase_replay_process(kbase_jd_atom *katom)
 999 {
1000         kbase_context *kctx = katom->kctx;
1001         kbase_jd_context *jctx = &kctx->jctx;
1002         mali_bool need_to_try_schedule_context = MALI_FALSE;
1003         base_jd_atom_v2 t_atom, f_atom;
1004         kbase_jd_atom *t_katom, *f_katom;
1005         struct device *dev = kctx->kbdev->dev;
1006
1007         if (katom->event_code == BASE_JD_EVENT_DONE) {
1008                 KBASE_LOG(2, dev, "Previous job succeeded - not replaying\n");
1009                 return MALI_REPLAY_STATUS_COMPLETE;
1010         }
1011
1012         if (jctx->sched_info.ctx.is_dying) {
1013                 KBASE_LOG(2, dev, "Not replaying; context is dying\n");
1014                 return MALI_REPLAY_STATUS_COMPLETE;
1015         }
1016
1017         dev_warn(dev, "Replaying jobs retry=%d\n", katom->retry_count);
1018
1019         katom->retry_count++;
1020         if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) {
1021                 dev_err(dev, "Replay exceeded limit - failing jobs\n");
1022                 /* katom->event_code is already set to the failure code of the
1023                    previous job */
1024                 return MALI_REPLAY_STATUS_COMPLETE;
1025         }
1026
1027         if (kbasep_replay_create_atoms(kctx, &t_atom, &f_atom,
1028                                        katom->nice_prio) != MALI_ERROR_NONE) {
1029                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1030                 return MALI_REPLAY_STATUS_COMPLETE;
1031         }
1032
1033         t_katom = &jctx->atoms[t_atom.atom_number];
1034         f_katom = &jctx->atoms[f_atom.atom_number];
1035
1036         if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) !=
1037                                                              MALI_ERROR_NONE) {
1038                 kbasep_release_katom(kctx, t_atom.atom_number);
1039                 kbasep_release_katom(kctx, f_atom.atom_number);
1040                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1041                 return MALI_REPLAY_STATUS_COMPLETE;
1042         }
1043
1044         kbasep_replay_reset_softjob(katom, f_katom);
1045
1046         need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom);
1047         if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) {
1048                 dev_err(dev, "Replay failed to submit atom\n");
1049                 kbasep_release_katom(kctx, f_atom.atom_number);
1050                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1051                 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
1052                 return MALI_REPLAY_STATUS_COMPLETE;
1053         }
1054         need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom);
1055         if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) {
1056                 dev_err(dev, "Replay failed to submit atom\n");
1057                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1058                 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
1059                 return MALI_REPLAY_STATUS_COMPLETE;
1060         }
1061
1062         katom->event_code = BASE_JD_EVENT_DONE;
1063
1064         if (need_to_try_schedule_context)
1065                 return MALI_REPLAY_STATUS_REPLAYING |
1066                                                 MALI_REPLAY_FLAG_JS_RESCHED;
1067         return MALI_REPLAY_STATUS_REPLAYING;
1068 }
1069