drivers/gpu/arm/midgard/mali_kbase_replay.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /**
  19  * @file mali_kbase_replay.c
  20  * Replay soft job handlers
  21  */
  22
  23 #include <linux/dma-mapping.h>
  24 #include <mali_kbase_config.h>
  25 #include <mali_kbase.h>
  26 #include <mali_kbase_mem.h>
  27 #include <mali_kbase_mem_linux.h>
  28
  29 #define JOB_NOT_STARTED 0
  30 #define JOB_TYPE_MASK      0xfe
  31 #define JOB_TYPE_NULL      (1 << 1)
  32 #define JOB_TYPE_VERTEX    (5 << 1)
  33 #define JOB_TYPE_TILER     (7 << 1)
  34 #define JOB_TYPE_FUSED     (8 << 1)
  35 #define JOB_TYPE_FRAGMENT  (9 << 1)
  36
  37 #define JOB_FLAG_DESC_SIZE           (1 << 0)
  38 #define JOB_FLAG_PERFORM_JOB_BARRIER (1 << 8)
  39
  40 #define JOB_HEADER_32_FBD_OFFSET (31*4)
  41 #define JOB_HEADER_64_FBD_OFFSET (44*4)
  42
  43 #define FBD_POINTER_MASK (~0x3f)
  44
  45 #define SFBD_TILER_OFFSET (48*4)
  46
  47 #define MFBD_TILER_OFFSET       (14*4)
  48
  49 #define FBD_HIERARCHY_WEIGHTS 8
  50 #define FBD_HIERARCHY_MASK_MASK 0x1fff
  51
  52 #define FBD_TYPE 1
  53
  54 #define HIERARCHY_WEIGHTS 13
  55
  56 #define JOB_HEADER_ID_MAX                 0xffff
  57
  58 #define JOB_SOURCE_ID(status)           (((status) >> 16) & 0xFFFF)
  59 #define JOB_POLYGON_LIST                (0x03)
  60
  61 struct job_head {
  62         u32 status;
  63         u32 not_complete_index;
  64         u64 fault_addr;
  65         u16 flags;
  66         u16 index;
  67         u16 dependencies[2];
  68         union {
  69                 u64 _64;
  70                 u32 _32;
  71         } next;
  72         u32 x[2];
  73         union {
  74                 u64 _64;
  75                 u32 _32;
  76         } fragment_fbd;
  77 };
  78
  79 static void dump_job_head(struct kbase_context *kctx, char *head_str,
  80                 struct job_head *job)
  81 {
  82 #ifdef CONFIG_MALI_DEBUG
  83         dev_dbg(kctx->kbdev->dev, "%s\n", head_str);
  84         dev_dbg(kctx->kbdev->dev, "addr               = %p\n"
  85                         "status             = %x\n"
  86                         "not_complete_index = %x\n"
  87                         "fault_addr         = %llx\n"
  88                         "flags              = %x\n"
  89                         "index              = %x\n"
  90                         "dependencies       = %x,%x\n",
  91                         job, job->status, job->not_complete_index,
  92                         job->fault_addr, job->flags, job->index,
  93                         job->dependencies[0],
  94                         job->dependencies[1]);
  95
  96         if (job->flags & JOB_FLAG_DESC_SIZE)
  97                 dev_dbg(kctx->kbdev->dev, "next               = %llx\n",
  98                                 job->next._64);
  99         else
 100                 dev_dbg(kctx->kbdev->dev, "next               = %x\n",
 101                                 job->next._32);
 102 #endif
 103 }
 104
 105 static int kbasep_replay_reset_sfbd(struct kbase_context *kctx,
 106                 u64 fbd_address, u64 tiler_heap_free,
 107                 u16 hierarchy_mask, u32 default_weight)
 108 {
 109         struct {
 110                 u32 padding_1[1];
 111                 u32 flags;
 112                 u64 padding_2[2];
 113                 u64 heap_free_address;
 114                 u32 padding[8];
 115                 u32 weights[FBD_HIERARCHY_WEIGHTS];
 116         } *fbd_tiler;
 117         struct kbase_vmap_struct map;
 118
 119         dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address);
 120
 121         fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET,
 122                         sizeof(*fbd_tiler), &map);
 123         if (!fbd_tiler) {
 124                 dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n");
 125                 return -EINVAL;
 126         }
 127
 128 #ifdef CONFIG_MALI_DEBUG
 129         dev_dbg(kctx->kbdev->dev,
 130                 "FBD tiler:\n"
 131                 "flags = %x\n"
 132                 "heap_free_address = %llx\n",
 133                 fbd_tiler->flags, fbd_tiler->heap_free_address);
 134 #endif
 135         if (hierarchy_mask) {
 136                 u32 weights[HIERARCHY_WEIGHTS];
 137                 u16 old_hierarchy_mask = fbd_tiler->flags &
 138                                                        FBD_HIERARCHY_MASK_MASK;
 139                 int i, j = 0;
 140
 141                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 142                         if (old_hierarchy_mask & (1 << i)) {
 143                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 144                                 weights[i] = fbd_tiler->weights[j++];
 145                         } else {
 146                                 weights[i] = default_weight;
 147                         }
 148                 }
 149
 150
 151                 dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x  New hierarchy mask=%x\n",
 152                                 old_hierarchy_mask, hierarchy_mask);
 153
 154                 for (i = 0; i < HIERARCHY_WEIGHTS; i++)
 155                         dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n",
 156                                         i, weights[i]);
 157
 158                 j = 0;
 159
 160                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 161                         if (hierarchy_mask & (1 << i)) {
 162                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 163
 164                                 dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n",
 165                                                 i, weights[i], j);
 166
 167                                 fbd_tiler->weights[j++] = weights[i];
 168                         }
 169                 }
 170
 171                 for (; j < FBD_HIERARCHY_WEIGHTS; j++)
 172                         fbd_tiler->weights[j] = 0;
 173
 174                 fbd_tiler->flags = hierarchy_mask | (1 << 16);
 175         }
 176
 177         fbd_tiler->heap_free_address = tiler_heap_free;
 178
 179         dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n",
 180                         fbd_tiler->heap_free_address, fbd_tiler->flags);
 181
 182         kbase_vunmap(kctx, &map);
 183
 184         return 0;
 185 }
 186
 187 static int kbasep_replay_reset_mfbd(struct kbase_context *kctx,
 188                 u64 fbd_address, u64 tiler_heap_free,
 189                 u16 hierarchy_mask, u32 default_weight)
 190 {
 191         struct kbase_vmap_struct map;
 192         struct {
 193                 u32 padding_0;
 194                 u32 flags;
 195                 u64 padding_1[2];
 196                 u64 heap_free_address;
 197                 u64 padding_2;
 198                 u32 weights[FBD_HIERARCHY_WEIGHTS];
 199         } *fbd_tiler;
 200
 201         dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address);
 202
 203         fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET,
 204                         sizeof(*fbd_tiler), &map);
 205         if (!fbd_tiler) {
 206                 dev_err(kctx->kbdev->dev,
 207                                "kbasep_replay_reset_fbd: failed to map fbd\n");
 208                 return -EINVAL;
 209         }
 210
 211 #ifdef CONFIG_MALI_DEBUG
 212         dev_dbg(kctx->kbdev->dev, "FBD tiler:\n"
 213                         "flags = %x\n"
 214                         "heap_free_address = %llx\n",
 215                         fbd_tiler->flags,
 216                         fbd_tiler->heap_free_address);
 217 #endif
 218         if (hierarchy_mask) {
 219                 u32 weights[HIERARCHY_WEIGHTS];
 220                 u16 old_hierarchy_mask = (fbd_tiler->flags) &
 221                                                        FBD_HIERARCHY_MASK_MASK;
 222                 int i, j = 0;
 223
 224                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 225                         if (old_hierarchy_mask & (1 << i)) {
 226                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 227                                 weights[i] = fbd_tiler->weights[j++];
 228                         } else {
 229                                 weights[i] = default_weight;
 230                         }
 231                 }
 232
 233
 234                 dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x  New hierarchy mask=%x\n",
 235                                 old_hierarchy_mask, hierarchy_mask);
 236
 237                 for (i = 0; i < HIERARCHY_WEIGHTS; i++)
 238                         dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n",
 239                                         i, weights[i]);
 240
 241                 j = 0;
 242
 243                 for (i = 0; i < HIERARCHY_WEIGHTS; i++) {
 244                         if (hierarchy_mask & (1 << i)) {
 245                                 KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS);
 246
 247                                 dev_dbg(kctx->kbdev->dev,
 248                                 " Writing hierarchy level %02d (%08x) to %d\n",
 249                                                              i, weights[i], j);
 250
 251                                 fbd_tiler->weights[j++] = weights[i];
 252                         }
 253                 }
 254
 255                 for (; j < FBD_HIERARCHY_WEIGHTS; j++)
 256                         fbd_tiler->weights[j] = 0;
 257
 258                 fbd_tiler->flags = hierarchy_mask | (1 << 16);
 259         }
 260
 261         fbd_tiler->heap_free_address = tiler_heap_free;
 262
 263         kbase_vunmap(kctx, &map);
 264
 265         return 0;
 266 }
 267
 268 /**
 269  * @brief Reset the status of an FBD pointed to by a tiler job
 270  *
 271  * This performs two functions :
 272  * - Set the hierarchy mask
 273  * - Reset the tiler free heap address
 274  *
 275  * @param[in] kctx              Context pointer
 276  * @param[in] job_header        Address of job header to reset.
 277  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 278  * @param[in] hierarchy_mask    The hierarchy mask to use
 279  * @param[in] default_weight    Default hierarchy weight to write when no other
 280  *                              weight is given in the FBD
 281  * @param[in] job_64            true if this job is using 64-bit
 282  *                              descriptors
 283  *
 284  * @return 0 on success, error code on failure
 285  */
 286 static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx,
 287                 u64 job_header, u64 tiler_heap_free,
 288                 u16 hierarchy_mask, u32 default_weight, bool job_64)
 289 {
 290         struct kbase_vmap_struct map;
 291         u64 fbd_address;
 292
 293         if (job_64) {
 294                 u64 *job_ext;
 295
 296                 job_ext = kbase_vmap(kctx,
 297                                 job_header + JOB_HEADER_64_FBD_OFFSET,
 298                                 sizeof(*job_ext), &map);
 299
 300                 if (!job_ext) {
 301                         dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n");
 302                         return -EINVAL;
 303                 }
 304
 305                 fbd_address = *job_ext;
 306
 307                 kbase_vunmap(kctx, &map);
 308         } else {
 309                 u32 *job_ext;
 310
 311                 job_ext = kbase_vmap(kctx,
 312                                 job_header + JOB_HEADER_32_FBD_OFFSET,
 313                                 sizeof(*job_ext), &map);
 314
 315                 if (!job_ext) {
 316                         dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n");
 317                         return -EINVAL;
 318                 }
 319
 320                 fbd_address = *job_ext;
 321
 322                 kbase_vunmap(kctx, &map);
 323         }
 324
 325         if (fbd_address & FBD_TYPE) {
 326                 return kbasep_replay_reset_mfbd(kctx,
 327                                                 fbd_address & FBD_POINTER_MASK,
 328                                                 tiler_heap_free,
 329                                                 hierarchy_mask,
 330                                                 default_weight);
 331         } else {
 332                 return kbasep_replay_reset_sfbd(kctx,
 333                                                 fbd_address & FBD_POINTER_MASK,
 334                                                 tiler_heap_free,
 335                                                 hierarchy_mask,
 336                                                 default_weight);
 337         }
 338 }
 339
 340 /**
 341  * @brief Reset the status of a job
 342  *
 343  * This performs the following functions :
 344  *
 345  * - Reset the Job Status field of each job to NOT_STARTED.
 346  * - Set the Job Type field of any Vertex Jobs to Null Job.
 347  * - For any jobs using an FBD, set the Tiler Heap Free field to the value of
 348  *   the tiler_heap_free parameter, and set the hierarchy level mask to the
 349  *   hier_mask parameter.
 350  * - Offset HW dependencies by the hw_job_id_offset parameter
 351  * - Set the Perform Job Barrier flag if this job is the first in the chain
 352  * - Read the address of the next job header
 353  *
 354  * @param[in] kctx              Context pointer
 355  * @param[in,out] job_header    Address of job header to reset. Set to address
 356  *                              of next job header on exit.
 357  * @param[in] prev_jc           Previous job chain to link to, if this job is
 358  *                              the last in the chain.
 359  * @param[in] hw_job_id_offset  Offset for HW job IDs
 360  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 361  * @param[in] hierarchy_mask    The hierarchy mask to use
 362  * @param[in] default_weight    Default hierarchy weight to write when no other
 363  *                              weight is given in the FBD
 364  * @param[in] first_in_chain    true if this job is the first in the chain
 365  * @param[in] fragment_chain    true if this job is in the fragment chain
 366  *
 367  * @return 0 on success, error code on failure
 368  */
 369 static int kbasep_replay_reset_job(struct kbase_context *kctx,
 370                 u64 *job_header, u64 prev_jc,
 371                 u64 tiler_heap_free, u16 hierarchy_mask,
 372                 u32 default_weight, u16 hw_job_id_offset,
 373                 bool first_in_chain, bool fragment_chain)
 374 {
 375         struct job_head *job;
 376         u64 new_job_header;
 377         struct kbase_vmap_struct map;
 378
 379         job = kbase_vmap(kctx, *job_header, sizeof(*job), &map);
 380         if (!job) {
 381                 dev_err(kctx->kbdev->dev,
 382                                  "kbasep_replay_parse_jc: failed to map jc\n");
 383                 return -EINVAL;
 384         }
 385
 386         dump_job_head(kctx, "Job header:", job);
 387
 388         if (job->status == JOB_NOT_STARTED && !fragment_chain) {
 389                 dev_err(kctx->kbdev->dev, "Job already not started\n");
 390                 goto out_unmap;
 391         }
 392         job->status = JOB_NOT_STARTED;
 393
 394         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_VERTEX)
 395                 job->flags = (job->flags & ~JOB_TYPE_MASK) | JOB_TYPE_NULL;
 396
 397         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_FUSED) {
 398                 dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n");
 399                 goto out_unmap;
 400         }
 401
 402         if (first_in_chain)
 403                 job->flags |= JOB_FLAG_PERFORM_JOB_BARRIER;
 404
 405         if ((job->dependencies[0] + hw_job_id_offset) > JOB_HEADER_ID_MAX ||
 406             (job->dependencies[1] + hw_job_id_offset) > JOB_HEADER_ID_MAX ||
 407             (job->index + hw_job_id_offset) > JOB_HEADER_ID_MAX) {
 408                 dev_err(kctx->kbdev->dev,
 409                              "Job indicies/dependencies out of valid range\n");
 410                 goto out_unmap;
 411         }
 412
 413         if (job->dependencies[0])
 414                 job->dependencies[0] += hw_job_id_offset;
 415         if (job->dependencies[1])
 416                 job->dependencies[1] += hw_job_id_offset;
 417
 418         job->index += hw_job_id_offset;
 419
 420         if (job->flags & JOB_FLAG_DESC_SIZE) {
 421                 new_job_header = job->next._64;
 422                 if (!job->next._64)
 423                         job->next._64 = prev_jc;
 424         } else {
 425                 new_job_header = job->next._32;
 426                 if (!job->next._32)
 427                         job->next._32 = prev_jc;
 428         }
 429         dump_job_head(kctx, "Updated to:", job);
 430
 431         if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_TILER) {
 432                 bool job_64 = (job->flags & JOB_FLAG_DESC_SIZE) != 0;
 433
 434                 if (kbasep_replay_reset_tiler_job(kctx, *job_header,
 435                                 tiler_heap_free, hierarchy_mask,
 436                                 default_weight, job_64) != 0)
 437                         goto out_unmap;
 438
 439         } else if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_FRAGMENT) {
 440                 u64 fbd_address;
 441
 442                 if (job->flags & JOB_FLAG_DESC_SIZE)
 443                         fbd_address = job->fragment_fbd._64;
 444                 else
 445                         fbd_address = (u64)job->fragment_fbd._32;
 446
 447                 if (fbd_address & FBD_TYPE) {
 448                         if (kbasep_replay_reset_mfbd(kctx,
 449                                         fbd_address & FBD_POINTER_MASK,
 450                                         tiler_heap_free,
 451                                         hierarchy_mask,
 452                                         default_weight) != 0)
 453                                 goto out_unmap;
 454                 } else {
 455                         if (kbasep_replay_reset_sfbd(kctx,
 456                                         fbd_address & FBD_POINTER_MASK,
 457                                         tiler_heap_free,
 458                                         hierarchy_mask,
 459                                         default_weight) != 0)
 460                                 goto out_unmap;
 461                 }
 462         }
 463
 464         kbase_vunmap(kctx, &map);
 465
 466         *job_header = new_job_header;
 467
 468         return 0;
 469
 470 out_unmap:
 471         kbase_vunmap(kctx, &map);
 472         return -EINVAL;
 473 }
 474
 475 /**
 476  * @brief Find the highest job ID in a job chain
 477  *
 478  * @param[in] kctx        Context pointer
 479  * @param[in] jc          Job chain start address
 480  * @param[out] hw_job_id  Highest job ID in chain
 481  *
 482  * @return 0 on success, error code on failure
 483  */
 484 static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx,
 485                 u64 jc, u16 *hw_job_id)
 486 {
 487         while (jc) {
 488                 struct job_head *job;
 489                 struct kbase_vmap_struct map;
 490
 491                 dev_dbg(kctx->kbdev->dev,
 492                         "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc);
 493
 494                 job = kbase_vmap(kctx, jc, sizeof(*job), &map);
 495                 if (!job) {
 496                         dev_err(kctx->kbdev->dev, "failed to map jc\n");
 497
 498                         return -EINVAL;
 499                 }
 500
 501                 if (job->index > *hw_job_id)
 502                         *hw_job_id = job->index;
 503
 504                 if (job->flags & JOB_FLAG_DESC_SIZE)
 505                         jc = job->next._64;
 506                 else
 507                         jc = job->next._32;
 508
 509                 kbase_vunmap(kctx, &map);
 510         }
 511
 512         return 0;
 513 }
 514
 515 /**
 516  * @brief Reset the status of a number of jobs
 517  *
 518  * This function walks the provided job chain, and calls
 519  * kbasep_replay_reset_job for each job. It also links the job chain to the
 520  * provided previous job chain.
 521  *
 522  * The function will fail if any of the jobs passed already have status of
 523  * NOT_STARTED.
 524  *
 525  * @param[in] kctx              Context pointer
 526  * @param[in] jc                Job chain to be processed
 527  * @param[in] prev_jc           Job chain to be added to. May be NULL
 528  * @param[in] tiler_heap_free   The value to reset Tiler Heap Free to
 529  * @param[in] hierarchy_mask    The hierarchy mask to use
 530  * @param[in] default_weight    Default hierarchy weight to write when no other
 531  *                              weight is given in the FBD
 532  * @param[in] hw_job_id_offset  Offset for HW job IDs
 533  * @param[in] fragment_chain    true if this chain is the fragment chain
 534  *
 535  * @return 0 on success, error code otherwise
 536  */
 537 static int kbasep_replay_parse_jc(struct kbase_context *kctx,
 538                 u64 jc, u64 prev_jc,
 539                 u64 tiler_heap_free, u16 hierarchy_mask,
 540                 u32 default_weight, u16 hw_job_id_offset,
 541                 bool fragment_chain)
 542 {
 543         bool first_in_chain = true;
 544         int nr_jobs = 0;
 545
 546         dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n",
 547                         jc, hw_job_id_offset);
 548
 549         while (jc) {
 550                 dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc);
 551
 552                 if (kbasep_replay_reset_job(kctx, &jc, prev_jc,
 553                                 tiler_heap_free, hierarchy_mask,
 554                                 default_weight, hw_job_id_offset,
 555                                 first_in_chain, fragment_chain) != 0)
 556                         return -EINVAL;
 557
 558                 first_in_chain = false;
 559
 560                 nr_jobs++;
 561                 if (fragment_chain &&
 562                     nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) {
 563                         dev_err(kctx->kbdev->dev,
 564                                 "Exceeded maximum number of jobs in fragment chain\n");
 565                         return -EINVAL;
 566                 }
 567         }
 568
 569         return 0;
 570 }
 571
 572 /**
 573  * @brief Reset the status of a replay job, and set up dependencies
 574  *
 575  * This performs the actions to allow the replay job to be re-run following
 576  * completion of the passed dependency.
 577  *
 578  * @param[in] katom     The atom to be reset
 579  * @param[in] dep_atom  The dependency to be attached to the atom
 580  */
 581 static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom,
 582                 struct kbase_jd_atom *dep_atom)
 583 {
 584         katom->status = KBASE_JD_ATOM_STATE_QUEUED;
 585         kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA);
 586         list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]);
 587 }
 588
 589 /**
 590  * @brief Allocate an unused katom
 591  *
 592  * This will search the provided context for an unused katom, and will mark it
 593  * as KBASE_JD_ATOM_STATE_QUEUED.
 594  *
 595  * If no atoms are available then the function will fail.
 596  *
 597  * @param[in] kctx      Context pointer
 598  * @return An atom ID, or -1 on failure
 599  */
 600 static int kbasep_allocate_katom(struct kbase_context *kctx)
 601 {
 602         struct kbase_jd_context *jctx = &kctx->jctx;
 603         int i;
 604
 605         for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) {
 606                 if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) {
 607                         jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED;
 608                         dev_dbg(kctx->kbdev->dev,
 609                                   "kbasep_allocate_katom: Allocated atom %d\n",
 610                                                                             i);
 611                         return i;
 612                 }
 613         }
 614
 615         return -1;
 616 }
 617
 618 /**
 619  * @brief Release a katom
 620  *
 621  * This will mark the provided atom as available, and remove any dependencies.
 622  *
 623  * For use on error path.
 624  *
 625  * @param[in] kctx      Context pointer
 626  * @param[in] atom_id   ID of atom to release
 627  */
 628 static void kbasep_release_katom(struct kbase_context *kctx, int atom_id)
 629 {
 630         struct kbase_jd_context *jctx = &kctx->jctx;
 631
 632         dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n",
 633                         atom_id);
 634
 635         while (!list_empty(&jctx->atoms[atom_id].dep_head[0]))
 636                 list_del(jctx->atoms[atom_id].dep_head[0].next);
 637
 638         while (!list_empty(&jctx->atoms[atom_id].dep_head[1]))
 639                 list_del(jctx->atoms[atom_id].dep_head[1].next);
 640
 641         jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED;
 642 }
 643
 644 static void kbasep_replay_create_atom(struct kbase_context *kctx,
 645                                       struct base_jd_atom_v2 *atom,
 646                                       int atom_nr,
 647                                       base_jd_prio prio)
 648 {
 649         atom->nr_extres = 0;
 650         atom->extres_list.value = NULL;
 651         atom->device_nr = 0;
 652         atom->prio = prio;
 653         atom->atom_number = atom_nr;
 654
 655         base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID);
 656         base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID);
 657
 658         atom->udata.blob[0] = 0;
 659         atom->udata.blob[1] = 0;
 660 }
 661
 662 /**
 663  * @brief Create two atoms for the purpose of replaying jobs
 664  *
 665  * Two atoms are allocated and created. The jc pointer is not set at this
 666  * stage. The second atom has a dependency on the first. The remaining fields
 667  * are set up as follows :
 668  *
 669  * - No external resources. Any required external resources will be held by the
 670  *   replay atom.
 671  * - device_nr is set to 0. This is not relevant as
 672  *   BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set.
 673  * - Priority is inherited from the replay job.
 674  *
 675  * @param[out] t_atom      Atom to use for tiler jobs
 676  * @param[out] f_atom      Atom to use for fragment jobs
 677  * @param[in]  prio        Priority of new atom (inherited from replay soft
 678  *                         job)
 679  * @return 0 on success, error code on failure
 680  */
 681 static int kbasep_replay_create_atoms(struct kbase_context *kctx,
 682                 struct base_jd_atom_v2 *t_atom,
 683                 struct base_jd_atom_v2 *f_atom,
 684                 base_jd_prio prio)
 685 {
 686         int t_atom_nr, f_atom_nr;
 687
 688         t_atom_nr = kbasep_allocate_katom(kctx);
 689         if (t_atom_nr < 0) {
 690                 dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
 691                 return -EINVAL;
 692         }
 693
 694         f_atom_nr = kbasep_allocate_katom(kctx);
 695         if (f_atom_nr < 0) {
 696                 dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
 697                 kbasep_release_katom(kctx, t_atom_nr);
 698                 return -EINVAL;
 699         }
 700
 701         kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio);
 702         kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio);
 703
 704         base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA);
 705
 706         return 0;
 707 }
 708
 709 #ifdef CONFIG_MALI_DEBUG
 710 static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload)
 711 {
 712         u64 next;
 713
 714         dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n");
 715         next = payload->tiler_jc_list;
 716
 717         while (next) {
 718                 struct kbase_vmap_struct map;
 719                 base_jd_replay_jc *jc_struct;
 720
 721                 jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map);
 722
 723                 if (!jc_struct)
 724                         return;
 725
 726                 dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n",
 727                                 jc_struct, jc_struct->jc, jc_struct->next);
 728
 729                 next = jc_struct->next;
 730
 731                 kbase_vunmap(kctx, &map);
 732         }
 733 }
 734 #endif
 735
 736 /**
 737  * @brief Parse a base_jd_replay_payload provided by userspace
 738  *
 739  * This will read the payload from userspace, and parse the job chains.
 740  *
 741  * @param[in] kctx         Context pointer
 742  * @param[in] replay_atom  Replay soft job atom
 743  * @param[in] t_atom       Atom to use for tiler jobs
 744  * @param[in] f_atom       Atom to use for fragment jobs
 745  * @return 0 on success, error code on failure
 746  */
 747 static int kbasep_replay_parse_payload(struct kbase_context *kctx,
 748                                               struct kbase_jd_atom *replay_atom,
 749                                               struct base_jd_atom_v2 *t_atom,
 750                                               struct base_jd_atom_v2 *f_atom)
 751 {
 752         base_jd_replay_payload *payload;
 753         u64 next;
 754         u64 prev_jc = 0;
 755         u16 hw_job_id_offset = 0;
 756         int ret = -EINVAL;
 757         struct kbase_vmap_struct map;
 758
 759         dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n",
 760                         replay_atom->jc, sizeof(payload));
 761
 762         payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map);
 763
 764         if (!payload) {
 765                 dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n");
 766                 return -EINVAL;
 767         }
 768
 769 #ifdef CONFIG_MALI_DEBUG
 770         dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload);
 771         dev_dbg(kctx->kbdev->dev, "Payload structure:\n"
 772                                   "tiler_jc_list            = %llx\n"
 773                                   "fragment_jc              = %llx\n"
 774                                   "tiler_heap_free          = %llx\n"
 775                                   "fragment_hierarchy_mask  = %x\n"
 776                                   "tiler_hierarchy_mask     = %x\n"
 777                                   "hierarchy_default_weight = %x\n"
 778                                   "tiler_core_req           = %x\n"
 779                                   "fragment_core_req        = %x\n",
 780                                                         payload->tiler_jc_list,
 781                                                           payload->fragment_jc,
 782                                                       payload->tiler_heap_free,
 783                                               payload->fragment_hierarchy_mask,
 784                                                  payload->tiler_hierarchy_mask,
 785                                              payload->hierarchy_default_weight,
 786                                                        payload->tiler_core_req,
 787                                                    payload->fragment_core_req);
 788         payload_dump(kctx, payload);
 789 #endif
 790
 791         t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER;
 792         f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER;
 793
 794         /* Sanity check core requirements*/
 795         if ((t_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
 796                                ~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_T ||
 797             (f_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
 798                               ~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_FS ||
 799              t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES ||
 800              f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
 801                 dev_err(kctx->kbdev->dev, "Invalid core requirements\n");
 802                 goto out;
 803         }
 804
 805         /* Process tiler job chains */
 806         next = payload->tiler_jc_list;
 807         if (!next) {
 808                 dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n");
 809                 goto out;
 810         }
 811
 812         while (next) {
 813                 base_jd_replay_jc *jc_struct;
 814                 struct kbase_vmap_struct jc_map;
 815                 u64 jc;
 816
 817                 jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map);
 818
 819                 if (!jc_struct) {
 820                         dev_err(kctx->kbdev->dev, "Failed to map jc struct\n");
 821                         goto out;
 822                 }
 823
 824                 jc = jc_struct->jc;
 825                 next = jc_struct->next;
 826                 if (next)
 827                         jc_struct->jc = 0;
 828
 829                 kbase_vunmap(kctx, &jc_map);
 830
 831                 if (jc) {
 832                         u16 max_hw_job_id = 0;
 833
 834                         if (kbasep_replay_find_hw_job_id(kctx, jc,
 835                                         &max_hw_job_id) != 0)
 836                                 goto out;
 837
 838                         if (kbasep_replay_parse_jc(kctx, jc, prev_jc,
 839                                         payload->tiler_heap_free,
 840                                         payload->tiler_hierarchy_mask,
 841                                         payload->hierarchy_default_weight,
 842                                         hw_job_id_offset, false) != 0) {
 843                                 goto out;
 844                         }
 845
 846                         hw_job_id_offset += max_hw_job_id;
 847
 848                         prev_jc = jc;
 849                 }
 850         }
 851         t_atom->jc = prev_jc;
 852
 853         /* Process fragment job chain */
 854         f_atom->jc = payload->fragment_jc;
 855         if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0,
 856                         payload->tiler_heap_free,
 857                         payload->fragment_hierarchy_mask,
 858                         payload->hierarchy_default_weight, 0,
 859                         true) != 0) {
 860                 goto out;
 861         }
 862
 863         if (!t_atom->jc || !f_atom->jc) {
 864                 dev_err(kctx->kbdev->dev, "Invalid payload\n");
 865                 goto out;
 866         }
 867
 868         dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n",
 869                         t_atom->jc, f_atom->jc);
 870         ret = 0;
 871
 872 out:
 873         kbase_vunmap(kctx, &map);
 874
 875         return ret;
 876 }
 877
 878 static void kbase_replay_process_worker(struct work_struct *data)
 879 {
 880         struct kbase_jd_atom *katom;
 881         struct kbase_context *kctx;
 882         struct kbase_jd_context *jctx;
 883         bool need_to_try_schedule_context = false;
 884
 885         struct base_jd_atom_v2 t_atom, f_atom;
 886         struct kbase_jd_atom *t_katom, *f_katom;
 887         base_jd_prio atom_prio;
 888
 889         katom = container_of(data, struct kbase_jd_atom, work);
 890         kctx = katom->kctx;
 891         jctx = &kctx->jctx;
 892
 893         mutex_lock(&jctx->lock);
 894
 895         atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority);
 896
 897         if (kbasep_replay_create_atoms(
 898                         kctx, &t_atom, &f_atom, atom_prio) != 0) {
 899                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 900                 goto out;
 901         }
 902
 903         t_katom = &jctx->atoms[t_atom.atom_number];
 904         f_katom = &jctx->atoms[f_atom.atom_number];
 905
 906         if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) {
 907                 kbasep_release_katom(kctx, t_atom.atom_number);
 908                 kbasep_release_katom(kctx, f_atom.atom_number);
 909                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 910                 goto out;
 911         }
 912
 913         kbasep_replay_reset_softjob(katom, f_katom);
 914
 915         need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom);
 916         if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) {
 917                 dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n");
 918                 kbasep_release_katom(kctx, f_atom.atom_number);
 919                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 920                 goto out;
 921         }
 922         need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom);
 923         if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) {
 924                 dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n");
 925                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 926                 goto out;
 927         }
 928
 929         katom->event_code = BASE_JD_EVENT_DONE;
 930
 931 out:
 932         if (katom->event_code != BASE_JD_EVENT_DONE) {
 933                 kbase_disjoint_state_down(kctx->kbdev);
 934
 935                 need_to_try_schedule_context |= jd_done_nolock(katom);
 936         }
 937
 938         if (need_to_try_schedule_context)
 939                 kbase_js_sched_all(kctx->kbdev);
 940
 941         mutex_unlock(&jctx->lock);
 942 }
 943
 944 /**
 945  * @brief Check job replay fault
 946  *
 947  * This will read the job payload, checks fault type and source, then decides
 948  * whether replay is required.
 949  *
 950  * @param[in] katom       The atom to be processed
 951  * @return  true (success) if replay required or false on failure.
 952  */
 953 static bool kbase_replay_fault_check(struct kbase_jd_atom *katom)
 954 {
 955         struct kbase_context *kctx = katom->kctx;
 956         struct device *dev = kctx->kbdev->dev;
 957         base_jd_replay_payload *payload;
 958         u64 job_header;
 959         u64 job_loop_detect;
 960         struct job_head *job;
 961         struct kbase_vmap_struct job_map;
 962         struct kbase_vmap_struct map;
 963         bool err = false;
 964
 965         /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or
 966          * if force_replay is enabled.
 967          */
 968         if (BASE_JD_EVENT_TERMINATED == katom->event_code) {
 969                 return false;
 970         } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) {
 971                 return true;
 972         } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) {
 973                 katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT;
 974                 return true;
 975         } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) {
 976                 /* No replay for faults of type other than
 977                  * BASE_JD_EVENT_DATA_INVALID_FAULT.
 978                  */
 979                 return false;
 980         }
 981
 982         /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc
 983          * to find out whether the source of exception is POLYGON_LIST. Replay
 984          * is required if the source of fault is POLYGON_LIST.
 985          */
 986         payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map);
 987         if (!payload) {
 988                 dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n");
 989                 return false;
 990         }
 991
 992 #ifdef CONFIG_MALI_DEBUG
 993         dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload);
 994         dev_dbg(dev, "\nPayload structure:\n"
 995                      "fragment_jc              = 0x%llx\n"
 996                      "fragment_hierarchy_mask  = 0x%x\n"
 997                      "fragment_core_req        = 0x%x\n",
 998                      payload->fragment_jc,
 999                      payload->fragment_hierarchy_mask,
1000                      payload->fragment_core_req);
1001 #endif
1002         /* Process fragment job chain */
1003         job_header      = (u64) payload->fragment_jc;
1004         job_loop_detect = job_header;
1005         while (job_header) {
1006                 job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map);
1007                 if (!job) {
1008                         dev_err(dev, "failed to map jc\n");
1009                         /* unmap payload*/
1010                         kbase_vunmap(kctx, &map);
1011                         return false;
1012                 }
1013
1014
1015 #ifdef CONFIG_MALI_DEBUG
1016                 dev_dbg(dev, "\njob_head structure:\n"
1017                              "Source ID:0x%x Access:0x%x Exception:0x%x\n"
1018                              "at job addr               = %p\n"
1019                              "not_complete_index        = 0x%x\n"
1020                              "fault_addr                = 0x%llx\n"
1021                              "flags                     = 0x%x\n"
1022                              "index                     = 0x%x\n"
1023                              "dependencies              = 0x%x,0x%x\n",
1024                              JOB_SOURCE_ID(job->status),
1025                              ((job->status >> 8) & 0x3),
1026                              (job->status  & 0xFF),
1027                              job,
1028                              job->not_complete_index,
1029                              job->fault_addr,
1030                              job->flags,
1031                              job->index,
1032                              job->dependencies[0],
1033                              job->dependencies[1]);
1034 #endif
1035
1036                 /* Replay only when the polygon list reader caused the
1037                  * DATA_INVALID_FAULT */
1038                 if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) &&
1039                     (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->status))) {
1040                         err = true;
1041                         kbase_vunmap(kctx, &job_map);
1042                         break;
1043                 }
1044
1045                 /* Move on to next fragment job in the list */
1046                 if (job->flags & JOB_FLAG_DESC_SIZE)
1047                         job_header = job->next._64;
1048                 else
1049                         job_header = job->next._32;
1050
1051                 kbase_vunmap(kctx, &job_map);
1052
1053                 /* Job chain loop detected */
1054                 if (job_header == job_loop_detect)
1055                         break;
1056         }
1057
1058         /* unmap payload*/
1059         kbase_vunmap(kctx, &map);
1060
1061         return err;
1062 }
1063
1064
1065 /**
1066  * @brief Process a replay job
1067  *
1068  * Called from kbase_process_soft_job.
1069  *
1070  * On exit, if the job has completed, katom->event_code will have been updated.
1071  * If the job has not completed, and is replaying jobs, then the atom status
1072  * will have been reset to KBASE_JD_ATOM_STATE_QUEUED.
1073  *
1074  * @param[in] katom  The atom to be processed
1075  * @return           false if the atom has completed
1076  *                   true if the atom is replaying jobs
1077  */
1078 bool kbase_replay_process(struct kbase_jd_atom *katom)
1079 {
1080         struct kbase_context *kctx = katom->kctx;
1081         struct kbase_jd_context *jctx = &kctx->jctx;
1082
1083         if (katom->event_code == BASE_JD_EVENT_DONE) {
1084                 dev_dbg(kctx->kbdev->dev, "Previous job succeeded - not replaying\n");
1085
1086                 if (katom->retry_count)
1087                         kbase_disjoint_state_down(kctx->kbdev);
1088
1089                 return false;
1090         }
1091
1092         if (jctx->sched_info.ctx.is_dying) {
1093                 dev_dbg(kctx->kbdev->dev, "Not replaying; context is dying\n");
1094
1095                 if (katom->retry_count)
1096                         kbase_disjoint_state_down(kctx->kbdev);
1097
1098                 return false;
1099         }
1100
1101         /* Check job exception type and source before replaying. */
1102         if (!kbase_replay_fault_check(katom)) {
1103                 dev_dbg(kctx->kbdev->dev,
1104                         "Replay cancelled on event %x\n", katom->event_code);
1105                 /* katom->event_code is already set to the failure code of the
1106                  * previous job.
1107                  */
1108                 return false;
1109         }
1110
1111         dev_warn(kctx->kbdev->dev, "Replaying jobs retry=%d\n",
1112                         katom->retry_count);
1113
1114         katom->retry_count++;
1115
1116         if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) {
1117                 dev_err(kctx->kbdev->dev, "Replay exceeded limit - failing jobs\n");
1118
1119                 kbase_disjoint_state_down(kctx->kbdev);
1120
1121                 /* katom->event_code is already set to the failure code of the
1122                    previous job */
1123                 return false;
1124         }
1125
1126         /* only enter the disjoint state once for the whole time while the replay is ongoing */
1127         if (katom->retry_count == 1)
1128                 kbase_disjoint_state_up(kctx->kbdev);
1129
1130         INIT_WORK(&katom->work, kbase_replay_process_worker);
1131         queue_work(kctx->event_workq, &katom->work);
1132
1133         return true;
1134 }