drivers/gpu/drm/radeon/radeon_kfd.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/module.h>
  24 #include <linux/fdtable.h>
  25 #include <linux/uaccess.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "cikd.h"
  29 #include "cik_reg.h"
  30 #include "radeon_kfd.h"
  31 #include "radeon_ucode.h"
  32 #include <linux/firmware.h>
  33
  34 #define CIK_PIPE_PER_MEC        (4)
  35
  36 struct kgd_mem {
  37         struct radeon_bo *bo;
  38         uint64_t gpu_addr;
  39         void *cpu_ptr;
  40 };
  41
  42
  43 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
  44                         void **mem_obj, uint64_t *gpu_addr,
  45                         void **cpu_ptr);
  46
  47 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
  48
  49 static uint64_t get_vmem_size(struct kgd_dev *kgd);
  50 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
  51
  52 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
  53 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
  54
  55 /*
  56  * Register access functions
  57  */
  58
  59 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
  60                 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
  61                 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
  62
  63 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
  64                                         unsigned int vmid);
  65
  66 static int kgd_init_memory(struct kgd_dev *kgd);
  67
  68 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
  69                                 uint32_t hpd_size, uint64_t hpd_gpu_addr);
  70
  71 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
  72                         uint32_t queue_id, uint32_t __user *wptr);
  73 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
  74 static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
  75                                 uint32_t pipe_id, uint32_t queue_id);
  76
  77 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
  78                                 unsigned int timeout, uint32_t pipe_id,
  79                                 uint32_t queue_id);
  80 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
  81 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
  82                                 unsigned int timeout);
  83
  84 static const struct kfd2kgd_calls kfd2kgd = {
  85         .init_gtt_mem_allocation = alloc_gtt_mem,
  86         .free_gtt_mem = free_gtt_mem,
  87         .get_vmem_size = get_vmem_size,
  88         .get_gpu_clock_counter = get_gpu_clock_counter,
  89         .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
  90         .program_sh_mem_settings = kgd_program_sh_mem_settings,
  91         .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
  92         .init_memory = kgd_init_memory,
  93         .init_pipeline = kgd_init_pipeline,
  94         .hqd_load = kgd_hqd_load,
  95         .hqd_sdma_load = kgd_hqd_sdma_load,
  96         .hqd_is_occupies = kgd_hqd_is_occupies,
  97         .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
  98         .hqd_destroy = kgd_hqd_destroy,
  99         .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
 100         .get_fw_version = get_fw_version
 101 };
 102
 103 static const struct kgd2kfd_calls *kgd2kfd;
 104
 105 bool radeon_kfd_init(void)
 106 {
 107         bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*,
 108                                 const struct kgd2kfd_calls**);
 109
 110         kgd2kfd_init_p = symbol_request(kgd2kfd_init);
 111
 112         if (kgd2kfd_init_p == NULL)
 113                 return false;
 114
 115         if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) {
 116                 symbol_put(kgd2kfd_init);
 117                 kgd2kfd = NULL;
 118
 119                 return false;
 120         }
 121
 122         return true;
 123 }
 124
 125 void radeon_kfd_fini(void)
 126 {
 127         if (kgd2kfd) {
 128                 kgd2kfd->exit();
 129                 symbol_put(kgd2kfd_init);
 130         }
 131 }
 132
 133 void radeon_kfd_device_probe(struct radeon_device *rdev)
 134 {
 135         if (kgd2kfd)
 136                 rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev);
 137 }
 138
 139 void radeon_kfd_device_init(struct radeon_device *rdev)
 140 {
 141         if (rdev->kfd) {
 142                 struct kgd2kfd_shared_resources gpu_resources = {
 143                         .compute_vmid_bitmap = 0xFF00,
 144
 145                         .first_compute_pipe = 1,
 146                         .compute_pipe_count = 8 - 1,
 147                 };
 148
 149                 radeon_doorbell_get_kfd_info(rdev,
 150                                 &gpu_resources.doorbell_physical_address,
 151                                 &gpu_resources.doorbell_aperture_size,
 152                                 &gpu_resources.doorbell_start_offset);
 153
 154                 kgd2kfd->device_init(rdev->kfd, &gpu_resources);
 155         }
 156 }
 157
 158 void radeon_kfd_device_fini(struct radeon_device *rdev)
 159 {
 160         if (rdev->kfd) {
 161                 kgd2kfd->device_exit(rdev->kfd);
 162                 rdev->kfd = NULL;
 163         }
 164 }
 165
 166 void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
 167 {
 168         if (rdev->kfd)
 169                 kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
 170 }
 171
 172 void radeon_kfd_suspend(struct radeon_device *rdev)
 173 {
 174         if (rdev->kfd)
 175                 kgd2kfd->suspend(rdev->kfd);
 176 }
 177
 178 int radeon_kfd_resume(struct radeon_device *rdev)
 179 {
 180         int r = 0;
 181
 182         if (rdev->kfd)
 183                 r = kgd2kfd->resume(rdev->kfd);
 184
 185         return r;
 186 }
 187
 188 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 189                         void **mem_obj, uint64_t *gpu_addr,
 190                         void **cpu_ptr)
 191 {
 192         struct radeon_device *rdev = (struct radeon_device *)kgd;
 193         struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
 194         int r;
 195
 196         BUG_ON(kgd == NULL);
 197         BUG_ON(gpu_addr == NULL);
 198         BUG_ON(cpu_ptr == NULL);
 199
 200         *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
 201         if ((*mem) == NULL)
 202                 return -ENOMEM;
 203
 204         r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
 205                                 RADEON_GEM_GTT_WC, NULL, NULL, &(*mem)->bo);
 206         if (r) {
 207                 dev_err(rdev->dev,
 208                         "failed to allocate BO for amdkfd (%d)\n", r);
 209                 return r;
 210         }
 211
 212         /* map the buffer */
 213         r = radeon_bo_reserve((*mem)->bo, true);
 214         if (r) {
 215                 dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
 216                 goto allocate_mem_reserve_bo_failed;
 217         }
 218
 219         r = radeon_bo_pin((*mem)->bo, RADEON_GEM_DOMAIN_GTT,
 220                                 &(*mem)->gpu_addr);
 221         if (r) {
 222                 dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r);
 223                 goto allocate_mem_pin_bo_failed;
 224         }
 225         *gpu_addr = (*mem)->gpu_addr;
 226
 227         r = radeon_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
 228         if (r) {
 229                 dev_err(rdev->dev,
 230                         "(%d) failed to map bo to kernel for amdkfd\n", r);
 231                 goto allocate_mem_kmap_bo_failed;
 232         }
 233         *cpu_ptr = (*mem)->cpu_ptr;
 234
 235         radeon_bo_unreserve((*mem)->bo);
 236
 237         return 0;
 238
 239 allocate_mem_kmap_bo_failed:
 240         radeon_bo_unpin((*mem)->bo);
 241 allocate_mem_pin_bo_failed:
 242         radeon_bo_unreserve((*mem)->bo);
 243 allocate_mem_reserve_bo_failed:
 244         radeon_bo_unref(&(*mem)->bo);
 245
 246         return r;
 247 }
 248
 249 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
 250 {
 251         struct kgd_mem *mem = (struct kgd_mem *) mem_obj;
 252
 253         BUG_ON(mem == NULL);
 254
 255         radeon_bo_reserve(mem->bo, true);
 256         radeon_bo_kunmap(mem->bo);
 257         radeon_bo_unpin(mem->bo);
 258         radeon_bo_unreserve(mem->bo);
 259         radeon_bo_unref(&(mem->bo));
 260         kfree(mem);
 261 }
 262
 263 static uint64_t get_vmem_size(struct kgd_dev *kgd)
 264 {
 265         struct radeon_device *rdev = (struct radeon_device *)kgd;
 266
 267         BUG_ON(kgd == NULL);
 268
 269         return rdev->mc.real_vram_size;
 270 }
 271
 272 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
 273 {
 274         struct radeon_device *rdev = (struct radeon_device *)kgd;
 275
 276         return rdev->asic->get_gpu_clock_counter(rdev);
 277 }
 278
 279 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
 280 {
 281         struct radeon_device *rdev = (struct radeon_device *)kgd;
 282
 283         /* The sclk is in quantas of 10kHz */
 284         return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
 285 }
 286
 287 static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
 288 {
 289         return (struct radeon_device *)kgd;
 290 }
 291
 292 static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
 293 {
 294         struct radeon_device *rdev = get_radeon_device(kgd);
 295
 296         writel(value, (void __iomem *)(rdev->rmmio + offset));
 297 }
 298
 299 static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
 300 {
 301         struct radeon_device *rdev = get_radeon_device(kgd);
 302
 303         return readl((void __iomem *)(rdev->rmmio + offset));
 304 }
 305
 306 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
 307                         uint32_t queue, uint32_t vmid)
 308 {
 309         struct radeon_device *rdev = get_radeon_device(kgd);
 310         uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
 311
 312         mutex_lock(&rdev->srbm_mutex);
 313         write_register(kgd, SRBM_GFX_CNTL, value);
 314 }
 315
 316 static void unlock_srbm(struct kgd_dev *kgd)
 317 {
 318         struct radeon_device *rdev = get_radeon_device(kgd);
 319
 320         write_register(kgd, SRBM_GFX_CNTL, 0);
 321         mutex_unlock(&rdev->srbm_mutex);
 322 }
 323
 324 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
 325                                 uint32_t queue_id)
 326 {
 327         uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
 328         uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
 329
 330         lock_srbm(kgd, mec, pipe, queue_id, 0);
 331 }
 332
 333 static void release_queue(struct kgd_dev *kgd)
 334 {
 335         unlock_srbm(kgd);
 336 }
 337
 338 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
 339                                         uint32_t sh_mem_config,
 340                                         uint32_t sh_mem_ape1_base,
 341                                         uint32_t sh_mem_ape1_limit,
 342                                         uint32_t sh_mem_bases)
 343 {
 344         lock_srbm(kgd, 0, 0, 0, vmid);
 345
 346         write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
 347         write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
 348         write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
 349         write_register(kgd, SH_MEM_BASES, sh_mem_bases);
 350
 351         unlock_srbm(kgd);
 352 }
 353
 354 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 355                                         unsigned int vmid)
 356 {
 357         /*
 358          * We have to assume that there is no outstanding mapping.
 359          * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
 360          * because a mapping is in progress or because a mapping finished and
 361          * the SW cleared it.
 362          * So the protocol is to always wait & clear.
 363          */
 364         uint32_t pasid_mapping = (pasid == 0) ? 0 :
 365                                 (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
 366
 367         write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
 368                         pasid_mapping);
 369
 370         while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
 371                                                                 (1U << vmid)))
 372                 cpu_relax();
 373         write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
 374
 375         return 0;
 376 }
 377
 378 static int kgd_init_memory(struct kgd_dev *kgd)
 379 {
 380         /*
 381          * Configure apertures:
 382          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
 383          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
 384          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
 385          */
 386         int i;
 387         uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000);
 388
 389         for (i = 8; i < 16; i++) {
 390                 uint32_t sh_mem_config;
 391
 392                 lock_srbm(kgd, 0, 0, 0, i);
 393
 394                 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
 395                 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
 396
 397                 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
 398
 399                 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
 400
 401                 /* Scratch aperture is not supported for now. */
 402                 write_register(kgd, SH_STATIC_MEM_CONFIG, 0);
 403
 404                 /* APE1 disabled for now. */
 405                 write_register(kgd, SH_MEM_APE1_BASE, 1);
 406                 write_register(kgd, SH_MEM_APE1_LIMIT, 0);
 407
 408                 unlock_srbm(kgd);
 409         }
 410
 411         return 0;
 412 }
 413
 414 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
 415                                 uint32_t hpd_size, uint64_t hpd_gpu_addr)
 416 {
 417         uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
 418         uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
 419
 420         lock_srbm(kgd, mec, pipe, 0, 0);
 421         write_register(kgd, CP_HPD_EOP_BASE_ADDR,
 422                         lower_32_bits(hpd_gpu_addr >> 8));
 423         write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
 424                         upper_32_bits(hpd_gpu_addr >> 8));
 425         write_register(kgd, CP_HPD_EOP_VMID, 0);
 426         write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
 427         unlock_srbm(kgd);
 428
 429         return 0;
 430 }
 431
 432 static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
 433 {
 434         uint32_t retval;
 435
 436         retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
 437                         m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
 438
 439         pr_debug("kfd: sdma base address: 0x%x\n", retval);
 440
 441         return retval;
 442 }
 443
 444 static inline struct cik_mqd *get_mqd(void *mqd)
 445 {
 446         return (struct cik_mqd *)mqd;
 447 }
 448
 449 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 450 {
 451         return (struct cik_sdma_rlc_registers *)mqd;
 452 }
 453
 454 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 455                         uint32_t queue_id, uint32_t __user *wptr)
 456 {
 457         uint32_t wptr_shadow, is_wptr_shadow_valid;
 458         struct cik_mqd *m;
 459
 460         m = get_mqd(mqd);
 461
 462         is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
 463
 464         acquire_queue(kgd, pipe_id, queue_id);
 465         write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
 466         write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
 467         write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
 468
 469         write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
 470         write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
 471         write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
 472
 473         write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
 474         write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
 475         write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
 476
 477         write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
 478
 479         write_register(kgd, CP_HQD_PERSISTENT_STATE,
 480                         m->cp_hqd_persistent_state);
 481         write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
 482         write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
 483
 484         write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
 485                         m->cp_hqd_atomic0_preop_lo);
 486
 487         write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
 488                         m->cp_hqd_atomic0_preop_hi);
 489
 490         write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
 491                         m->cp_hqd_atomic1_preop_lo);
 492
 493         write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
 494                         m->cp_hqd_atomic1_preop_hi);
 495
 496         write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
 497                         m->cp_hqd_pq_rptr_report_addr_lo);
 498
 499         write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
 500                         m->cp_hqd_pq_rptr_report_addr_hi);
 501
 502         write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
 503
 504         write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
 505                         m->cp_hqd_pq_wptr_poll_addr_lo);
 506
 507         write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
 508                         m->cp_hqd_pq_wptr_poll_addr_hi);
 509
 510         write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
 511                         m->cp_hqd_pq_doorbell_control);
 512
 513         write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
 514
 515         write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
 516
 517         write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
 518         write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
 519
 520         write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
 521
 522         if (is_wptr_shadow_valid)
 523                 write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
 524
 525         write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
 526         release_queue(kgd);
 527
 528         return 0;
 529 }
 530
 531 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
 532 {
 533         struct cik_sdma_rlc_registers *m;
 534         uint32_t sdma_base_addr;
 535
 536         m = get_sdma_mqd(mqd);
 537         sdma_base_addr = get_sdma_base_addr(m);
 538
 539         write_register(kgd,
 540                         sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR,
 541                         m->sdma_rlc_virtual_addr);
 542
 543         write_register(kgd,
 544                         sdma_base_addr + SDMA0_RLC0_RB_BASE,
 545                         m->sdma_rlc_rb_base);
 546
 547         write_register(kgd,
 548                         sdma_base_addr + SDMA0_RLC0_RB_BASE_HI,
 549                         m->sdma_rlc_rb_base_hi);
 550
 551         write_register(kgd,
 552                         sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO,
 553                         m->sdma_rlc_rb_rptr_addr_lo);
 554
 555         write_register(kgd,
 556                         sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI,
 557                         m->sdma_rlc_rb_rptr_addr_hi);
 558
 559         write_register(kgd,
 560                         sdma_base_addr + SDMA0_RLC0_DOORBELL,
 561                         m->sdma_rlc_doorbell);
 562
 563         write_register(kgd,
 564                         sdma_base_addr + SDMA0_RLC0_RB_CNTL,
 565                         m->sdma_rlc_rb_cntl);
 566
 567         return 0;
 568 }
 569
 570 static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
 571                                 uint32_t pipe_id, uint32_t queue_id)
 572 {
 573         uint32_t act;
 574         bool retval = false;
 575         uint32_t low, high;
 576
 577         acquire_queue(kgd, pipe_id, queue_id);
 578         act = read_register(kgd, CP_HQD_ACTIVE);
 579         if (act) {
 580                 low = lower_32_bits(queue_address >> 8);
 581                 high = upper_32_bits(queue_address >> 8);
 582
 583                 if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
 584                                 high == read_register(kgd, CP_HQD_PQ_BASE_HI))
 585                         retval = true;
 586         }
 587         release_queue(kgd);
 588         return retval;
 589 }
 590
 591 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 592 {
 593         struct cik_sdma_rlc_registers *m;
 594         uint32_t sdma_base_addr;
 595         uint32_t sdma_rlc_rb_cntl;
 596
 597         m = get_sdma_mqd(mqd);
 598         sdma_base_addr = get_sdma_base_addr(m);
 599
 600         sdma_rlc_rb_cntl = read_register(kgd,
 601                                         sdma_base_addr + SDMA0_RLC0_RB_CNTL);
 602
 603         if (sdma_rlc_rb_cntl & SDMA_RB_ENABLE)
 604                 return true;
 605
 606         return false;
 607 }
 608
 609 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
 610                                 unsigned int timeout, uint32_t pipe_id,
 611                                 uint32_t queue_id)
 612 {
 613         uint32_t temp;
 614
 615         acquire_queue(kgd, pipe_id, queue_id);
 616         write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
 617
 618         write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
 619
 620         while (true) {
 621                 temp = read_register(kgd, CP_HQD_ACTIVE);
 622                 if (temp & 0x1)
 623                         break;
 624                 if (timeout == 0) {
 625                         pr_err("kfd: cp queue preemption time out (%dms)\n",
 626                                 temp);
 627                         return -ETIME;
 628                 }
 629                 msleep(20);
 630                 timeout -= 20;
 631         }
 632
 633         release_queue(kgd);
 634         return 0;
 635 }
 636
 637 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 638                                 unsigned int timeout)
 639 {
 640         struct cik_sdma_rlc_registers *m;
 641         uint32_t sdma_base_addr;
 642         uint32_t temp;
 643
 644         m = get_sdma_mqd(mqd);
 645         sdma_base_addr = get_sdma_base_addr(m);
 646
 647         temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);
 648         temp = temp & ~SDMA_RB_ENABLE;
 649         write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);
 650
 651         while (true) {
 652                 temp = read_register(kgd, sdma_base_addr +
 653                                                 SDMA0_RLC0_CONTEXT_STATUS);
 654                 if (temp & SDMA_RLC_IDLE)
 655                         break;
 656                 if (timeout == 0)
 657                         return -ETIME;
 658                 msleep(20);
 659                 timeout -= 20;
 660         }
 661
 662         write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);
 663         write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);
 664         write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);
 665         write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);
 666
 667         return 0;
 668 }
 669
 670 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 671 {
 672         struct radeon_device *rdev = (struct radeon_device *) kgd;
 673         const union radeon_firmware_header *hdr;
 674
 675         BUG_ON(kgd == NULL || rdev->mec_fw == NULL);
 676
 677         switch (type) {
 678         case KGD_ENGINE_PFP:
 679                 hdr = (const union radeon_firmware_header *) rdev->pfp_fw->data;
 680                 break;
 681
 682         case KGD_ENGINE_ME:
 683                 hdr = (const union radeon_firmware_header *) rdev->me_fw->data;
 684                 break;
 685
 686         case KGD_ENGINE_CE:
 687                 hdr = (const union radeon_firmware_header *) rdev->ce_fw->data;
 688                 break;
 689
 690         case KGD_ENGINE_MEC1:
 691                 hdr = (const union radeon_firmware_header *) rdev->mec_fw->data;
 692                 break;
 693
 694         case KGD_ENGINE_MEC2:
 695                 hdr = (const union radeon_firmware_header *)
 696                                                         rdev->mec2_fw->data;
 697                 break;
 698
 699         case KGD_ENGINE_RLC:
 700                 hdr = (const union radeon_firmware_header *) rdev->rlc_fw->data;
 701                 break;
 702
 703         case KGD_ENGINE_SDMA:
 704                 hdr = (const union radeon_firmware_header *)
 705                                                         rdev->sdma_fw->data;
 706                 break;
 707
 708         default:
 709                 return 0;
 710         }
 711
 712         if (hdr == NULL)
 713                 return 0;
 714
 715         /* Only 12 bit in use*/
 716         return hdr->common.ucode_version;
 717 }