drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 207 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 208 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 209 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 210 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 211 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 212 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 213 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 214 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 215 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 216 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 217 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 218 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 219 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 220 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 221 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 222 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 223 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 224 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 225 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 226 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 227
 228
 229 static const u32 cayman_golden_registers2[] =
 230 {
 231         0x3e5c, 0xffffffff, 0x00000000,
 232         0x3e48, 0xffffffff, 0x00000000,
 233         0x3e4c, 0xffffffff, 0x00000000,
 234         0x3e64, 0xffffffff, 0x00000000,
 235         0x3e50, 0xffffffff, 0x00000000,
 236         0x3e60, 0xffffffff, 0x00000000
 237 };
 238
 239 static const u32 cayman_golden_registers[] =
 240 {
 241         0x5eb4, 0xffffffff, 0x00000002,
 242         0x5e78, 0x8f311ff1, 0x001000f0,
 243         0x3f90, 0xffff0000, 0xff000000,
 244         0x9148, 0xffff0000, 0xff000000,
 245         0x3f94, 0xffff0000, 0xff000000,
 246         0x914c, 0xffff0000, 0xff000000,
 247         0xc78, 0x00000080, 0x00000080,
 248         0xbd4, 0x70073777, 0x00011003,
 249         0xd02c, 0xbfffff1f, 0x08421000,
 250         0xd0b8, 0x73773777, 0x02011003,
 251         0x5bc0, 0x00200000, 0x50100000,
 252         0x98f8, 0x33773777, 0x02011003,
 253         0x98fc, 0xffffffff, 0x76541032,
 254         0x7030, 0x31000311, 0x00000011,
 255         0x2f48, 0x33773777, 0x42010001,
 256         0x6b28, 0x00000010, 0x00000012,
 257         0x7728, 0x00000010, 0x00000012,
 258         0x10328, 0x00000010, 0x00000012,
 259         0x10f28, 0x00000010, 0x00000012,
 260         0x11b28, 0x00000010, 0x00000012,
 261         0x12728, 0x00000010, 0x00000012,
 262         0x240c, 0x000007ff, 0x00000000,
 263         0x8a14, 0xf000001f, 0x00000007,
 264         0x8b24, 0x3fff3fff, 0x00ff0fff,
 265         0x8b10, 0x0000ff0f, 0x00000000,
 266         0x28a4c, 0x07ffffff, 0x06000000,
 267         0x10c, 0x00000001, 0x00010003,
 268         0xa02c, 0xffffffff, 0x0000009b,
 269         0x913c, 0x0000010f, 0x01000100,
 270         0x8c04, 0xf8ff00ff, 0x40600060,
 271         0x28350, 0x00000f01, 0x00000000,
 272         0x9508, 0x3700001f, 0x00000002,
 273         0x960c, 0xffffffff, 0x54763210,
 274         0x88c4, 0x001f3ae3, 0x00000082,
 275         0x88d0, 0xffffffff, 0x0f40df40,
 276         0x88d4, 0x0000001f, 0x00000010,
 277         0x8974, 0xffffffff, 0x00000000
 278 };
 279
 280 static const u32 dvst_golden_registers2[] =
 281 {
 282         0x8f8, 0xffffffff, 0,
 283         0x8fc, 0x00380000, 0,
 284         0x8f8, 0xffffffff, 1,
 285         0x8fc, 0x0e000000, 0
 286 };
 287
 288 static const u32 dvst_golden_registers[] =
 289 {
 290         0x690, 0x3fff3fff, 0x20c00033,
 291         0x918c, 0x0fff0fff, 0x00010006,
 292         0x91a8, 0x0fff0fff, 0x00010006,
 293         0x9150, 0xffffdfff, 0x6e944040,
 294         0x917c, 0x0fff0fff, 0x00030002,
 295         0x9198, 0x0fff0fff, 0x00030002,
 296         0x915c, 0x0fff0fff, 0x00010000,
 297         0x3f90, 0xffff0001, 0xff000000,
 298         0x9178, 0x0fff0fff, 0x00070000,
 299         0x9194, 0x0fff0fff, 0x00070000,
 300         0x9148, 0xffff0001, 0xff000000,
 301         0x9190, 0x0fff0fff, 0x00090008,
 302         0x91ac, 0x0fff0fff, 0x00090008,
 303         0x3f94, 0xffff0000, 0xff000000,
 304         0x914c, 0xffff0000, 0xff000000,
 305         0x929c, 0x00000fff, 0x00000001,
 306         0x55e4, 0xff607fff, 0xfc000100,
 307         0x8a18, 0xff000fff, 0x00000100,
 308         0x8b28, 0xff000fff, 0x00000100,
 309         0x9144, 0xfffc0fff, 0x00000100,
 310         0x6ed8, 0x00010101, 0x00010000,
 311         0x9830, 0xffffffff, 0x00000000,
 312         0x9834, 0xf00fffff, 0x00000400,
 313         0x9838, 0xfffffffe, 0x00000000,
 314         0xd0c0, 0xff000fff, 0x00000100,
 315         0xd02c, 0xbfffff1f, 0x08421000,
 316         0xd0b8, 0x73773777, 0x12010001,
 317         0x5bb0, 0x000000f0, 0x00000070,
 318         0x98f8, 0x73773777, 0x12010001,
 319         0x98fc, 0xffffffff, 0x00000010,
 320         0x9b7c, 0x00ff0000, 0x00fc0000,
 321         0x8030, 0x00001f0f, 0x0000100a,
 322         0x2f48, 0x73773777, 0x12010001,
 323         0x2408, 0x00030000, 0x000c007f,
 324         0x8a14, 0xf000003f, 0x00000007,
 325         0x8b24, 0x3fff3fff, 0x00ff0fff,
 326         0x8b10, 0x0000ff0f, 0x00000000,
 327         0x28a4c, 0x07ffffff, 0x06000000,
 328         0x4d8, 0x00000fff, 0x00000100,
 329         0xa008, 0xffffffff, 0x00010000,
 330         0x913c, 0xffff03ff, 0x01000100,
 331         0x8c00, 0x000000ff, 0x00000003,
 332         0x8c04, 0xf8ff00ff, 0x40600060,
 333         0x8cf0, 0x1fff1fff, 0x08e00410,
 334         0x28350, 0x00000f01, 0x00000000,
 335         0x9508, 0xf700071f, 0x00000002,
 336         0x960c, 0xffffffff, 0x54763210,
 337         0x20ef8, 0x01ff01ff, 0x00000002,
 338         0x20e98, 0xfffffbff, 0x00200000,
 339         0x2015c, 0xffffffff, 0x00000f40,
 340         0x88c4, 0x001f3ae3, 0x00000082,
 341         0x8978, 0x3fffffff, 0x04050140,
 342         0x88d4, 0x0000001f, 0x00000010,
 343         0x8974, 0xffffffff, 0x00000000
 344 };
 345
 346 static const u32 scrapper_golden_registers[] =
 347 {
 348         0x690, 0x3fff3fff, 0x20c00033,
 349         0x918c, 0x0fff0fff, 0x00010006,
 350         0x918c, 0x0fff0fff, 0x00010006,
 351         0x91a8, 0x0fff0fff, 0x00010006,
 352         0x91a8, 0x0fff0fff, 0x00010006,
 353         0x9150, 0xffffdfff, 0x6e944040,
 354         0x9150, 0xffffdfff, 0x6e944040,
 355         0x917c, 0x0fff0fff, 0x00030002,
 356         0x917c, 0x0fff0fff, 0x00030002,
 357         0x9198, 0x0fff0fff, 0x00030002,
 358         0x9198, 0x0fff0fff, 0x00030002,
 359         0x915c, 0x0fff0fff, 0x00010000,
 360         0x915c, 0x0fff0fff, 0x00010000,
 361         0x3f90, 0xffff0001, 0xff000000,
 362         0x3f90, 0xffff0001, 0xff000000,
 363         0x9178, 0x0fff0fff, 0x00070000,
 364         0x9178, 0x0fff0fff, 0x00070000,
 365         0x9194, 0x0fff0fff, 0x00070000,
 366         0x9194, 0x0fff0fff, 0x00070000,
 367         0x9148, 0xffff0001, 0xff000000,
 368         0x9148, 0xffff0001, 0xff000000,
 369         0x9190, 0x0fff0fff, 0x00090008,
 370         0x9190, 0x0fff0fff, 0x00090008,
 371         0x91ac, 0x0fff0fff, 0x00090008,
 372         0x91ac, 0x0fff0fff, 0x00090008,
 373         0x3f94, 0xffff0000, 0xff000000,
 374         0x3f94, 0xffff0000, 0xff000000,
 375         0x914c, 0xffff0000, 0xff000000,
 376         0x914c, 0xffff0000, 0xff000000,
 377         0x929c, 0x00000fff, 0x00000001,
 378         0x929c, 0x00000fff, 0x00000001,
 379         0x55e4, 0xff607fff, 0xfc000100,
 380         0x8a18, 0xff000fff, 0x00000100,
 381         0x8a18, 0xff000fff, 0x00000100,
 382         0x8b28, 0xff000fff, 0x00000100,
 383         0x8b28, 0xff000fff, 0x00000100,
 384         0x9144, 0xfffc0fff, 0x00000100,
 385         0x9144, 0xfffc0fff, 0x00000100,
 386         0x6ed8, 0x00010101, 0x00010000,
 387         0x9830, 0xffffffff, 0x00000000,
 388         0x9830, 0xffffffff, 0x00000000,
 389         0x9834, 0xf00fffff, 0x00000400,
 390         0x9834, 0xf00fffff, 0x00000400,
 391         0x9838, 0xfffffffe, 0x00000000,
 392         0x9838, 0xfffffffe, 0x00000000,
 393         0xd0c0, 0xff000fff, 0x00000100,
 394         0xd02c, 0xbfffff1f, 0x08421000,
 395         0xd02c, 0xbfffff1f, 0x08421000,
 396         0xd0b8, 0x73773777, 0x12010001,
 397         0xd0b8, 0x73773777, 0x12010001,
 398         0x5bb0, 0x000000f0, 0x00000070,
 399         0x98f8, 0x73773777, 0x12010001,
 400         0x98f8, 0x73773777, 0x12010001,
 401         0x98fc, 0xffffffff, 0x00000010,
 402         0x98fc, 0xffffffff, 0x00000010,
 403         0x9b7c, 0x00ff0000, 0x00fc0000,
 404         0x9b7c, 0x00ff0000, 0x00fc0000,
 405         0x8030, 0x00001f0f, 0x0000100a,
 406         0x8030, 0x00001f0f, 0x0000100a,
 407         0x2f48, 0x73773777, 0x12010001,
 408         0x2f48, 0x73773777, 0x12010001,
 409         0x2408, 0x00030000, 0x000c007f,
 410         0x8a14, 0xf000003f, 0x00000007,
 411         0x8a14, 0xf000003f, 0x00000007,
 412         0x8b24, 0x3fff3fff, 0x00ff0fff,
 413         0x8b24, 0x3fff3fff, 0x00ff0fff,
 414         0x8b10, 0x0000ff0f, 0x00000000,
 415         0x8b10, 0x0000ff0f, 0x00000000,
 416         0x28a4c, 0x07ffffff, 0x06000000,
 417         0x28a4c, 0x07ffffff, 0x06000000,
 418         0x4d8, 0x00000fff, 0x00000100,
 419         0x4d8, 0x00000fff, 0x00000100,
 420         0xa008, 0xffffffff, 0x00010000,
 421         0xa008, 0xffffffff, 0x00010000,
 422         0x913c, 0xffff03ff, 0x01000100,
 423         0x913c, 0xffff03ff, 0x01000100,
 424         0x90e8, 0x001fffff, 0x010400c0,
 425         0x8c00, 0x000000ff, 0x00000003,
 426         0x8c00, 0x000000ff, 0x00000003,
 427         0x8c04, 0xf8ff00ff, 0x40600060,
 428         0x8c04, 0xf8ff00ff, 0x40600060,
 429         0x8c30, 0x0000000f, 0x00040005,
 430         0x8cf0, 0x1fff1fff, 0x08e00410,
 431         0x8cf0, 0x1fff1fff, 0x08e00410,
 432         0x900c, 0x00ffffff, 0x0017071f,
 433         0x28350, 0x00000f01, 0x00000000,
 434         0x28350, 0x00000f01, 0x00000000,
 435         0x9508, 0xf700071f, 0x00000002,
 436         0x9508, 0xf700071f, 0x00000002,
 437         0x9688, 0x00300000, 0x0017000f,
 438         0x960c, 0xffffffff, 0x54763210,
 439         0x960c, 0xffffffff, 0x54763210,
 440         0x20ef8, 0x01ff01ff, 0x00000002,
 441         0x20e98, 0xfffffbff, 0x00200000,
 442         0x2015c, 0xffffffff, 0x00000f40,
 443         0x88c4, 0x001f3ae3, 0x00000082,
 444         0x88c4, 0x001f3ae3, 0x00000082,
 445         0x8978, 0x3fffffff, 0x04050140,
 446         0x8978, 0x3fffffff, 0x04050140,
 447         0x88d4, 0x0000001f, 0x00000010,
 448         0x88d4, 0x0000001f, 0x00000010,
 449         0x8974, 0xffffffff, 0x00000000,
 450         0x8974, 0xffffffff, 0x00000000
 451 };
 452
 453 static void ni_init_golden_registers(struct radeon_device *rdev)
 454 {
 455         switch (rdev->family) {
 456         case CHIP_CAYMAN:
 457                 radeon_program_register_sequence(rdev,
 458                                                  cayman_golden_registers,
 459                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 460                 radeon_program_register_sequence(rdev,
 461                                                  cayman_golden_registers2,
 462                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 463                 break;
 464         case CHIP_ARUBA:
 465                 if ((rdev->pdev->device == 0x9900) ||
 466                     (rdev->pdev->device == 0x9901) ||
 467                     (rdev->pdev->device == 0x9903) ||
 468                     (rdev->pdev->device == 0x9904) ||
 469                     (rdev->pdev->device == 0x9905) ||
 470                     (rdev->pdev->device == 0x9906) ||
 471                     (rdev->pdev->device == 0x9907) ||
 472                     (rdev->pdev->device == 0x9908) ||
 473                     (rdev->pdev->device == 0x9909) ||
 474                     (rdev->pdev->device == 0x990A) ||
 475                     (rdev->pdev->device == 0x990B) ||
 476                     (rdev->pdev->device == 0x990C) ||
 477                     (rdev->pdev->device == 0x990D) ||
 478                     (rdev->pdev->device == 0x990E) ||
 479                     (rdev->pdev->device == 0x990F) ||
 480                     (rdev->pdev->device == 0x9910) ||
 481                     (rdev->pdev->device == 0x9913) ||
 482                     (rdev->pdev->device == 0x9917) ||
 483                     (rdev->pdev->device == 0x9918)) {
 484                         radeon_program_register_sequence(rdev,
 485                                                          dvst_golden_registers,
 486                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 487                         radeon_program_register_sequence(rdev,
 488                                                          dvst_golden_registers2,
 489                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 490                 } else {
 491                         radeon_program_register_sequence(rdev,
 492                                                          scrapper_golden_registers,
 493                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 494                         radeon_program_register_sequence(rdev,
 495                                                          dvst_golden_registers2,
 496                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 497                 }
 498                 break;
 499         default:
 500                 break;
 501         }
 502 }
 503
 504 #define BTC_IO_MC_REGS_SIZE 29
 505
 506 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 507         {0x00000077, 0xff010100},
 508         {0x00000078, 0x00000000},
 509         {0x00000079, 0x00001434},
 510         {0x0000007a, 0xcc08ec08},
 511         {0x0000007b, 0x00040000},
 512         {0x0000007c, 0x000080c0},
 513         {0x0000007d, 0x09000000},
 514         {0x0000007e, 0x00210404},
 515         {0x00000081, 0x08a8e800},
 516         {0x00000082, 0x00030444},
 517         {0x00000083, 0x00000000},
 518         {0x00000085, 0x00000001},
 519         {0x00000086, 0x00000002},
 520         {0x00000087, 0x48490000},
 521         {0x00000088, 0x20244647},
 522         {0x00000089, 0x00000005},
 523         {0x0000008b, 0x66030000},
 524         {0x0000008c, 0x00006603},
 525         {0x0000008d, 0x00000100},
 526         {0x0000008f, 0x00001c0a},
 527         {0x00000090, 0xff000001},
 528         {0x00000094, 0x00101101},
 529         {0x00000095, 0x00000fff},
 530         {0x00000096, 0x00116fff},
 531         {0x00000097, 0x60010000},
 532         {0x00000098, 0x10010000},
 533         {0x00000099, 0x00006000},
 534         {0x0000009a, 0x00001000},
 535         {0x0000009f, 0x00946a00}
 536 };
 537
 538 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 539         {0x00000077, 0xff010100},
 540         {0x00000078, 0x00000000},
 541         {0x00000079, 0x00001434},
 542         {0x0000007a, 0xcc08ec08},
 543         {0x0000007b, 0x00040000},
 544         {0x0000007c, 0x000080c0},
 545         {0x0000007d, 0x09000000},
 546         {0x0000007e, 0x00210404},
 547         {0x00000081, 0x08a8e800},
 548         {0x00000082, 0x00030444},
 549         {0x00000083, 0x00000000},
 550         {0x00000085, 0x00000001},
 551         {0x00000086, 0x00000002},
 552         {0x00000087, 0x48490000},
 553         {0x00000088, 0x20244647},
 554         {0x00000089, 0x00000005},
 555         {0x0000008b, 0x66030000},
 556         {0x0000008c, 0x00006603},
 557         {0x0000008d, 0x00000100},
 558         {0x0000008f, 0x00001c0a},
 559         {0x00000090, 0xff000001},
 560         {0x00000094, 0x00101101},
 561         {0x00000095, 0x00000fff},
 562         {0x00000096, 0x00116fff},
 563         {0x00000097, 0x60010000},
 564         {0x00000098, 0x10010000},
 565         {0x00000099, 0x00006000},
 566         {0x0000009a, 0x00001000},
 567         {0x0000009f, 0x00936a00}
 568 };
 569
 570 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 571         {0x00000077, 0xff010100},
 572         {0x00000078, 0x00000000},
 573         {0x00000079, 0x00001434},
 574         {0x0000007a, 0xcc08ec08},
 575         {0x0000007b, 0x00040000},
 576         {0x0000007c, 0x000080c0},
 577         {0x0000007d, 0x09000000},
 578         {0x0000007e, 0x00210404},
 579         {0x00000081, 0x08a8e800},
 580         {0x00000082, 0x00030444},
 581         {0x00000083, 0x00000000},
 582         {0x00000085, 0x00000001},
 583         {0x00000086, 0x00000002},
 584         {0x00000087, 0x48490000},
 585         {0x00000088, 0x20244647},
 586         {0x00000089, 0x00000005},
 587         {0x0000008b, 0x66030000},
 588         {0x0000008c, 0x00006603},
 589         {0x0000008d, 0x00000100},
 590         {0x0000008f, 0x00001c0a},
 591         {0x00000090, 0xff000001},
 592         {0x00000094, 0x00101101},
 593         {0x00000095, 0x00000fff},
 594         {0x00000096, 0x00116fff},
 595         {0x00000097, 0x60010000},
 596         {0x00000098, 0x10010000},
 597         {0x00000099, 0x00006000},
 598         {0x0000009a, 0x00001000},
 599         {0x0000009f, 0x00916a00}
 600 };
 601
 602 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 603         {0x00000077, 0xff010100},
 604         {0x00000078, 0x00000000},
 605         {0x00000079, 0x00001434},
 606         {0x0000007a, 0xcc08ec08},
 607         {0x0000007b, 0x00040000},
 608         {0x0000007c, 0x000080c0},
 609         {0x0000007d, 0x09000000},
 610         {0x0000007e, 0x00210404},
 611         {0x00000081, 0x08a8e800},
 612         {0x00000082, 0x00030444},
 613         {0x00000083, 0x00000000},
 614         {0x00000085, 0x00000001},
 615         {0x00000086, 0x00000002},
 616         {0x00000087, 0x48490000},
 617         {0x00000088, 0x20244647},
 618         {0x00000089, 0x00000005},
 619         {0x0000008b, 0x66030000},
 620         {0x0000008c, 0x00006603},
 621         {0x0000008d, 0x00000100},
 622         {0x0000008f, 0x00001c0a},
 623         {0x00000090, 0xff000001},
 624         {0x00000094, 0x00101101},
 625         {0x00000095, 0x00000fff},
 626         {0x00000096, 0x00116fff},
 627         {0x00000097, 0x60010000},
 628         {0x00000098, 0x10010000},
 629         {0x00000099, 0x00006000},
 630         {0x0000009a, 0x00001000},
 631         {0x0000009f, 0x00976b00}
 632 };
 633
 634 int ni_mc_load_microcode(struct radeon_device *rdev)
 635 {
 636         const __be32 *fw_data;
 637         u32 mem_type, running, blackout = 0;
 638         u32 *io_mc_regs;
 639         int i, ucode_size, regs_size;
 640
 641         if (!rdev->mc_fw)
 642                 return -EINVAL;
 643
 644         switch (rdev->family) {
 645         case CHIP_BARTS:
 646                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 647                 ucode_size = BTC_MC_UCODE_SIZE;
 648                 regs_size = BTC_IO_MC_REGS_SIZE;
 649                 break;
 650         case CHIP_TURKS:
 651                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 652                 ucode_size = BTC_MC_UCODE_SIZE;
 653                 regs_size = BTC_IO_MC_REGS_SIZE;
 654                 break;
 655         case CHIP_CAICOS:
 656         default:
 657                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 658                 ucode_size = BTC_MC_UCODE_SIZE;
 659                 regs_size = BTC_IO_MC_REGS_SIZE;
 660                 break;
 661         case CHIP_CAYMAN:
 662                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 663                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 664                 regs_size = BTC_IO_MC_REGS_SIZE;
 665                 break;
 666         }
 667
 668         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 669         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 670
 671         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 672                 if (running) {
 673                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 674                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 675                 }
 676
 677                 /* reset the engine and set to writable */
 678                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 679                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 680
 681                 /* load mc io regs */
 682                 for (i = 0; i < regs_size; i++) {
 683                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 684                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 685                 }
 686                 /* load the MC ucode */
 687                 fw_data = (const __be32 *)rdev->mc_fw->data;
 688                 for (i = 0; i < ucode_size; i++)
 689                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 690
 691                 /* put the engine back into the active state */
 692                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 693                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 694                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 695
 696                 /* wait for training to complete */
 697                 for (i = 0; i < rdev->usec_timeout; i++) {
 698                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 699                                 break;
 700                         udelay(1);
 701                 }
 702
 703                 if (running)
 704                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 705         }
 706
 707         return 0;
 708 }
 709
 710 int ni_init_microcode(struct radeon_device *rdev)
 711 {
 712         const char *chip_name;
 713         const char *rlc_chip_name;
 714         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 715         size_t smc_req_size = 0;
 716         char fw_name[30];
 717         int err;
 718
 719         DRM_DEBUG("\n");
 720
 721         switch (rdev->family) {
 722         case CHIP_BARTS:
 723                 chip_name = "BARTS";
 724                 rlc_chip_name = "BTC";
 725                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 726                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 727                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 728                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 729                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 730                 break;
 731         case CHIP_TURKS:
 732                 chip_name = "TURKS";
 733                 rlc_chip_name = "BTC";
 734                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 735                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 736                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 737                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 738                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 739                 break;
 740         case CHIP_CAICOS:
 741                 chip_name = "CAICOS";
 742                 rlc_chip_name = "BTC";
 743                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 744                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 745                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 746                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 747                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 748                 break;
 749         case CHIP_CAYMAN:
 750                 chip_name = "CAYMAN";
 751                 rlc_chip_name = "CAYMAN";
 752                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 753                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 754                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 755                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 756                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 757                 break;
 758         case CHIP_ARUBA:
 759                 chip_name = "ARUBA";
 760                 rlc_chip_name = "ARUBA";
 761                 /* pfp/me same size as CAYMAN */
 762                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 763                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 764                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 765                 mc_req_size = 0;
 766                 break;
 767         default: BUG();
 768         }
 769
 770         DRM_INFO("Loading %s Microcode\n", chip_name);
 771
 772         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 773         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 774         if (err)
 775                 goto out;
 776         if (rdev->pfp_fw->size != pfp_req_size) {
 777                 printk(KERN_ERR
 778                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 779                        rdev->pfp_fw->size, fw_name);
 780                 err = -EINVAL;
 781                 goto out;
 782         }
 783
 784         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 785         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 786         if (err)
 787                 goto out;
 788         if (rdev->me_fw->size != me_req_size) {
 789                 printk(KERN_ERR
 790                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 791                        rdev->me_fw->size, fw_name);
 792                 err = -EINVAL;
 793         }
 794
 795         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 796         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 797         if (err)
 798                 goto out;
 799         if (rdev->rlc_fw->size != rlc_req_size) {
 800                 printk(KERN_ERR
 801                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 802                        rdev->rlc_fw->size, fw_name);
 803                 err = -EINVAL;
 804         }
 805
 806         /* no MC ucode on TN */
 807         if (!(rdev->flags & RADEON_IS_IGP)) {
 808                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 809                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 810                 if (err)
 811                         goto out;
 812                 if (rdev->mc_fw->size != mc_req_size) {
 813                         printk(KERN_ERR
 814                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 815                                rdev->mc_fw->size, fw_name);
 816                         err = -EINVAL;
 817                 }
 818         }
 819
 820         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 821                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 822                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 823                 if (err) {
 824                         printk(KERN_ERR
 825                                "smc: error loading firmware \"%s\"\n",
 826                                fw_name);
 827                         release_firmware(rdev->smc_fw);
 828                         rdev->smc_fw = NULL;
 829                         err = 0;
 830                 } else if (rdev->smc_fw->size != smc_req_size) {
 831                         printk(KERN_ERR
 832                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 833                                rdev->mc_fw->size, fw_name);
 834                         err = -EINVAL;
 835                 }
 836         }
 837
 838 out:
 839         if (err) {
 840                 if (err != -EINVAL)
 841                         printk(KERN_ERR
 842                                "ni_cp: Failed to load firmware \"%s\"\n",
 843                                fw_name);
 844                 release_firmware(rdev->pfp_fw);
 845                 rdev->pfp_fw = NULL;
 846                 release_firmware(rdev->me_fw);
 847                 rdev->me_fw = NULL;
 848                 release_firmware(rdev->rlc_fw);
 849                 rdev->rlc_fw = NULL;
 850                 release_firmware(rdev->mc_fw);
 851                 rdev->mc_fw = NULL;
 852         }
 853         return err;
 854 }
 855
 856 /**
 857  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 858  *
 859  * @rdev: radeon_device pointer
 860  * @reg: register offset in bytes
 861  * @val: register value
 862  *
 863  * Returns 0 for success or -EINVAL for an invalid register
 864  *
 865  */
 866 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 867                                      u32 reg, u32 *val)
 868 {
 869         switch (reg) {
 870         case GRBM_STATUS:
 871         case GRBM_STATUS_SE0:
 872         case GRBM_STATUS_SE1:
 873         case SRBM_STATUS:
 874         case SRBM_STATUS2:
 875         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 876         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 877         case UVD_STATUS:
 878                 *val = RREG32(reg);
 879                 return 0;
 880         default:
 881                 return -EINVAL;
 882         }
 883 }
 884
 885 int tn_get_temp(struct radeon_device *rdev)
 886 {
 887         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 888         int actual_temp = (temp / 8) - 49;
 889
 890         return actual_temp * 1000;
 891 }
 892
 893 /*
 894  * Core functions
 895  */
 896 static void cayman_gpu_init(struct radeon_device *rdev)
 897 {
 898         u32 gb_addr_config = 0;
 899         u32 mc_shared_chmap, mc_arb_ramcfg;
 900         u32 cgts_tcc_disable;
 901         u32 sx_debug_1;
 902         u32 smx_dc_ctl0;
 903         u32 cgts_sm_ctrl_reg;
 904         u32 hdp_host_path_cntl;
 905         u32 tmp;
 906         u32 disabled_rb_mask;
 907         int i, j;
 908
 909         switch (rdev->family) {
 910         case CHIP_CAYMAN:
 911                 rdev->config.cayman.max_shader_engines = 2;
 912                 rdev->config.cayman.max_pipes_per_simd = 4;
 913                 rdev->config.cayman.max_tile_pipes = 8;
 914                 rdev->config.cayman.max_simds_per_se = 12;
 915                 rdev->config.cayman.max_backends_per_se = 4;
 916                 rdev->config.cayman.max_texture_channel_caches = 8;
 917                 rdev->config.cayman.max_gprs = 256;
 918                 rdev->config.cayman.max_threads = 256;
 919                 rdev->config.cayman.max_gs_threads = 32;
 920                 rdev->config.cayman.max_stack_entries = 512;
 921                 rdev->config.cayman.sx_num_of_sets = 8;
 922                 rdev->config.cayman.sx_max_export_size = 256;
 923                 rdev->config.cayman.sx_max_export_pos_size = 64;
 924                 rdev->config.cayman.sx_max_export_smx_size = 192;
 925                 rdev->config.cayman.max_hw_contexts = 8;
 926                 rdev->config.cayman.sq_num_cf_insts = 2;
 927
 928                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 929                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 930                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 931                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 932                 break;
 933         case CHIP_ARUBA:
 934         default:
 935                 rdev->config.cayman.max_shader_engines = 1;
 936                 rdev->config.cayman.max_pipes_per_simd = 4;
 937                 rdev->config.cayman.max_tile_pipes = 2;
 938                 if ((rdev->pdev->device == 0x9900) ||
 939                     (rdev->pdev->device == 0x9901) ||
 940                     (rdev->pdev->device == 0x9905) ||
 941                     (rdev->pdev->device == 0x9906) ||
 942                     (rdev->pdev->device == 0x9907) ||
 943                     (rdev->pdev->device == 0x9908) ||
 944                     (rdev->pdev->device == 0x9909) ||
 945                     (rdev->pdev->device == 0x990B) ||
 946                     (rdev->pdev->device == 0x990C) ||
 947                     (rdev->pdev->device == 0x990F) ||
 948                     (rdev->pdev->device == 0x9910) ||
 949                     (rdev->pdev->device == 0x9917) ||
 950                     (rdev->pdev->device == 0x9999) ||
 951                     (rdev->pdev->device == 0x999C)) {
 952                         rdev->config.cayman.max_simds_per_se = 6;
 953                         rdev->config.cayman.max_backends_per_se = 2;
 954                         rdev->config.cayman.max_hw_contexts = 8;
 955                         rdev->config.cayman.sx_max_export_size = 256;
 956                         rdev->config.cayman.sx_max_export_pos_size = 64;
 957                         rdev->config.cayman.sx_max_export_smx_size = 192;
 958                 } else if ((rdev->pdev->device == 0x9903) ||
 959                            (rdev->pdev->device == 0x9904) ||
 960                            (rdev->pdev->device == 0x990A) ||
 961                            (rdev->pdev->device == 0x990D) ||
 962                            (rdev->pdev->device == 0x990E) ||
 963                            (rdev->pdev->device == 0x9913) ||
 964                            (rdev->pdev->device == 0x9918) ||
 965                            (rdev->pdev->device == 0x999D)) {
 966                         rdev->config.cayman.max_simds_per_se = 4;
 967                         rdev->config.cayman.max_backends_per_se = 2;
 968                         rdev->config.cayman.max_hw_contexts = 8;
 969                         rdev->config.cayman.sx_max_export_size = 256;
 970                         rdev->config.cayman.sx_max_export_pos_size = 64;
 971                         rdev->config.cayman.sx_max_export_smx_size = 192;
 972                 } else if ((rdev->pdev->device == 0x9919) ||
 973                            (rdev->pdev->device == 0x9990) ||
 974                            (rdev->pdev->device == 0x9991) ||
 975                            (rdev->pdev->device == 0x9994) ||
 976                            (rdev->pdev->device == 0x9995) ||
 977                            (rdev->pdev->device == 0x9996) ||
 978                            (rdev->pdev->device == 0x999A) ||
 979                            (rdev->pdev->device == 0x99A0)) {
 980                         rdev->config.cayman.max_simds_per_se = 3;
 981                         rdev->config.cayman.max_backends_per_se = 1;
 982                         rdev->config.cayman.max_hw_contexts = 4;
 983                         rdev->config.cayman.sx_max_export_size = 128;
 984                         rdev->config.cayman.sx_max_export_pos_size = 32;
 985                         rdev->config.cayman.sx_max_export_smx_size = 96;
 986                 } else {
 987                         rdev->config.cayman.max_simds_per_se = 2;
 988                         rdev->config.cayman.max_backends_per_se = 1;
 989                         rdev->config.cayman.max_hw_contexts = 4;
 990                         rdev->config.cayman.sx_max_export_size = 128;
 991                         rdev->config.cayman.sx_max_export_pos_size = 32;
 992                         rdev->config.cayman.sx_max_export_smx_size = 96;
 993                 }
 994                 rdev->config.cayman.max_texture_channel_caches = 2;
 995                 rdev->config.cayman.max_gprs = 256;
 996                 rdev->config.cayman.max_threads = 256;
 997                 rdev->config.cayman.max_gs_threads = 32;
 998                 rdev->config.cayman.max_stack_entries = 512;
 999                 rdev->config.cayman.sx_num_of_sets = 8;
1000                 rdev->config.cayman.sq_num_cf_insts = 2;
1001
1002                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
1003                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
1004                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
1005                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
1006                 break;
1007         }
1008
1009         /* Initialize HDP */
1010         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1011                 WREG32((0x2c14 + j), 0x00000000);
1012                 WREG32((0x2c18 + j), 0x00000000);
1013                 WREG32((0x2c1c + j), 0x00000000);
1014                 WREG32((0x2c20 + j), 0x00000000);
1015                 WREG32((0x2c24 + j), 0x00000000);
1016         }
1017
1018         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1019         WREG32(SRBM_INT_CNTL, 0x1);
1020         WREG32(SRBM_INT_ACK, 0x1);
1021
1022         evergreen_fix_pci_max_read_req_size(rdev);
1023
1024         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1025         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1026
1027         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1028         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1029         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1030                 rdev->config.cayman.mem_row_size_in_kb = 4;
1031         /* XXX use MC settings? */
1032         rdev->config.cayman.shader_engine_tile_size = 32;
1033         rdev->config.cayman.num_gpus = 1;
1034         rdev->config.cayman.multi_gpu_tile_size = 64;
1035
1036         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1037         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1038         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1039         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1040         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1041         rdev->config.cayman.num_shader_engines = tmp + 1;
1042         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1043         rdev->config.cayman.num_gpus = tmp + 1;
1044         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1045         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1046         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1047         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1048
1049
1050         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1051          * not have bank info, so create a custom tiling dword.
1052          * bits 3:0   num_pipes
1053          * bits 7:4   num_banks
1054          * bits 11:8  group_size
1055          * bits 15:12 row_size
1056          */
1057         rdev->config.cayman.tile_config = 0;
1058         switch (rdev->config.cayman.num_tile_pipes) {
1059         case 1:
1060         default:
1061                 rdev->config.cayman.tile_config |= (0 << 0);
1062                 break;
1063         case 2:
1064                 rdev->config.cayman.tile_config |= (1 << 0);
1065                 break;
1066         case 4:
1067                 rdev->config.cayman.tile_config |= (2 << 0);
1068                 break;
1069         case 8:
1070                 rdev->config.cayman.tile_config |= (3 << 0);
1071                 break;
1072         }
1073
1074         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1075         if (rdev->flags & RADEON_IS_IGP)
1076                 rdev->config.cayman.tile_config |= 1 << 4;
1077         else {
1078                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1079                 case 0: /* four banks */
1080                         rdev->config.cayman.tile_config |= 0 << 4;
1081                         break;
1082                 case 1: /* eight banks */
1083                         rdev->config.cayman.tile_config |= 1 << 4;
1084                         break;
1085                 case 2: /* sixteen banks */
1086                 default:
1087                         rdev->config.cayman.tile_config |= 2 << 4;
1088                         break;
1089                 }
1090         }
1091         rdev->config.cayman.tile_config |=
1092                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1093         rdev->config.cayman.tile_config |=
1094                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1095
1096         tmp = 0;
1097         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1098                 u32 rb_disable_bitmap;
1099
1100                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1101                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1102                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1103                 tmp <<= 4;
1104                 tmp |= rb_disable_bitmap;
1105         }
1106         /* enabled rb are just the one not disabled :) */
1107         disabled_rb_mask = tmp;
1108         tmp = 0;
1109         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1110                 tmp |= (1 << i);
1111         /* if all the backends are disabled, fix it up here */
1112         if ((disabled_rb_mask & tmp) == tmp) {
1113                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1114                         disabled_rb_mask &= ~(1 << i);
1115         }
1116
1117         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1118                 u32 simd_disable_bitmap;
1119
1120                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1121                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1122                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1123                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1124                 tmp <<= 16;
1125                 tmp |= simd_disable_bitmap;
1126         }
1127         rdev->config.cayman.active_simds = hweight32(~tmp);
1128
1129         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1130         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1131
1132         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1133         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1134         if (ASIC_IS_DCE6(rdev))
1135                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1136         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1137         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1138         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1139         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1140         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1141         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1142
1143         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1144             (rdev->flags & RADEON_IS_IGP)) {
1145                 if ((disabled_rb_mask & 3) == 2) {
1146                         /* RB1 disabled, RB0 enabled */
1147                         tmp = 0x00000000;
1148                 } else {
1149                         /* RB0 disabled, RB1 enabled */
1150                         tmp = 0x11111111;
1151                 }
1152         } else {
1153                 tmp = gb_addr_config & NUM_PIPES_MASK;
1154                 tmp = r6xx_remap_render_backend(rdev, tmp,
1155                                                 rdev->config.cayman.max_backends_per_se *
1156                                                 rdev->config.cayman.max_shader_engines,
1157                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1158         }
1159         WREG32(GB_BACKEND_MAP, tmp);
1160
1161         cgts_tcc_disable = 0xffff0000;
1162         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1163                 cgts_tcc_disable &= ~(1 << (16 + i));
1164         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1165         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1166         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1167         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1168
1169         /* reprogram the shader complex */
1170         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1171         for (i = 0; i < 16; i++)
1172                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1173         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1174
1175         /* set HW defaults for 3D engine */
1176         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1177
1178         sx_debug_1 = RREG32(SX_DEBUG_1);
1179         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1180         WREG32(SX_DEBUG_1, sx_debug_1);
1181
1182         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1183         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1184         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1185         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1186
1187         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1188
1189         /* need to be explicitly zero-ed */
1190         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1191         WREG32(SQ_LSTMP_RING_BASE, 0);
1192         WREG32(SQ_HSTMP_RING_BASE, 0);
1193         WREG32(SQ_ESTMP_RING_BASE, 0);
1194         WREG32(SQ_GSTMP_RING_BASE, 0);
1195         WREG32(SQ_VSTMP_RING_BASE, 0);
1196         WREG32(SQ_PSTMP_RING_BASE, 0);
1197
1198         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1199
1200         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1201                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1202                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1203
1204         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1205                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1206                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1207
1208
1209         WREG32(VGT_NUM_INSTANCES, 1);
1210
1211         WREG32(CP_PERFMON_CNTL, 0);
1212
1213         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1214                                   FETCH_FIFO_HIWATER(0x4) |
1215                                   DONE_FIFO_HIWATER(0xe0) |
1216                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1217
1218         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1219         WREG32(SQ_CONFIG, (VC_ENABLE |
1220                            EXPORT_SRC_C |
1221                            GFX_PRIO(0) |
1222                            CS1_PRIO(0) |
1223                            CS2_PRIO(1)));
1224         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1225
1226         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1227                                           FORCE_EOV_MAX_REZ_CNT(255)));
1228
1229         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1230                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1231
1232         WREG32(VGT_GS_VERTEX_REUSE, 16);
1233         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1234
1235         WREG32(CB_PERF_CTR0_SEL_0, 0);
1236         WREG32(CB_PERF_CTR0_SEL_1, 0);
1237         WREG32(CB_PERF_CTR1_SEL_0, 0);
1238         WREG32(CB_PERF_CTR1_SEL_1, 0);
1239         WREG32(CB_PERF_CTR2_SEL_0, 0);
1240         WREG32(CB_PERF_CTR2_SEL_1, 0);
1241         WREG32(CB_PERF_CTR3_SEL_0, 0);
1242         WREG32(CB_PERF_CTR3_SEL_1, 0);
1243
1244         tmp = RREG32(HDP_MISC_CNTL);
1245         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1246         WREG32(HDP_MISC_CNTL, tmp);
1247
1248         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1249         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1250
1251         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1252
1253         udelay(50);
1254
1255         /* set clockgating golden values on TN */
1256         if (rdev->family == CHIP_ARUBA) {
1257                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1258                 tmp &= ~0x00380000;
1259                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1260                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1261                 tmp &= ~0x0e000000;
1262                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1263         }
1264 }
1265
1266 /*
1267  * GART
1268  */
1269 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1270 {
1271         /* flush hdp cache */
1272         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1273
1274         /* bits 0-7 are the VM contexts0-7 */
1275         WREG32(VM_INVALIDATE_REQUEST, 1);
1276 }
1277
1278 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1279 {
1280         int i, r;
1281
1282         if (rdev->gart.robj == NULL) {
1283                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1284                 return -EINVAL;
1285         }
1286         r = radeon_gart_table_vram_pin(rdev);
1287         if (r)
1288                 return r;
1289         /* Setup TLB control */
1290         WREG32(MC_VM_MX_L1_TLB_CNTL,
1291                (0xA << 7) |
1292                ENABLE_L1_TLB |
1293                ENABLE_L1_FRAGMENT_PROCESSING |
1294                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1295                ENABLE_ADVANCED_DRIVER_MODEL |
1296                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1297         /* Setup L2 cache */
1298         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1299                ENABLE_L2_FRAGMENT_PROCESSING |
1300                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1301                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1302                EFFECTIVE_L2_QUEUE_SIZE(7) |
1303                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1304         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1305         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1306                BANK_SELECT(6) |
1307                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1308         /* setup context0 */
1309         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1310         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1311         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1312         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1313                         (u32)(rdev->dummy_page.addr >> 12));
1314         WREG32(VM_CONTEXT0_CNTL2, 0);
1315         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1316                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1317
1318         WREG32(0x15D4, 0);
1319         WREG32(0x15D8, 0);
1320         WREG32(0x15DC, 0);
1321
1322         /* empty context1-7 */
1323         /* Assign the pt base to something valid for now; the pts used for
1324          * the VMs are determined by the application and setup and assigned
1325          * on the fly in the vm part of radeon_gart.c
1326          */
1327         for (i = 1; i < 8; i++) {
1328                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1329                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1330                         rdev->vm_manager.max_pfn - 1);
1331                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1332                        rdev->vm_manager.saved_table_addr[i]);
1333         }
1334
1335         /* enable context1-7 */
1336         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1337                (u32)(rdev->dummy_page.addr >> 12));
1338         WREG32(VM_CONTEXT1_CNTL2, 4);
1339         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1340                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1341                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1342                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1343                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1344                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1345                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1346                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1347                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1348                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1349                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1350                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1351                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1352                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1353
1354         cayman_pcie_gart_tlb_flush(rdev);
1355         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1356                  (unsigned)(rdev->mc.gtt_size >> 20),
1357                  (unsigned long long)rdev->gart.table_addr);
1358         rdev->gart.ready = true;
1359         return 0;
1360 }
1361
1362 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1363 {
1364         unsigned i;
1365
1366         for (i = 1; i < 8; ++i) {
1367                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1368                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1369         }
1370
1371         /* Disable all tables */
1372         WREG32(VM_CONTEXT0_CNTL, 0);
1373         WREG32(VM_CONTEXT1_CNTL, 0);
1374         /* Setup TLB control */
1375         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1376                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1377                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1378         /* Setup L2 cache */
1379         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1380                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1381                EFFECTIVE_L2_QUEUE_SIZE(7) |
1382                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1383         WREG32(VM_L2_CNTL2, 0);
1384         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1385                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1386         radeon_gart_table_vram_unpin(rdev);
1387 }
1388
1389 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1390 {
1391         cayman_pcie_gart_disable(rdev);
1392         radeon_gart_table_vram_free(rdev);
1393         radeon_gart_fini(rdev);
1394 }
1395
1396 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1397                               int ring, u32 cp_int_cntl)
1398 {
1399         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1400
1401         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1402         WREG32(CP_INT_CNTL, cp_int_cntl);
1403 }
1404
1405 /*
1406  * CP.
1407  */
1408 void cayman_fence_ring_emit(struct radeon_device *rdev,
1409                             struct radeon_fence *fence)
1410 {
1411         struct radeon_ring *ring = &rdev->ring[fence->ring];
1412         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1413         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1414                 PACKET3_SH_ACTION_ENA;
1415
1416         /* flush read cache over gart for this vmid */
1417         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1418         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1419         radeon_ring_write(ring, 0xFFFFFFFF);
1420         radeon_ring_write(ring, 0);
1421         radeon_ring_write(ring, 10); /* poll interval */
1422         /* EVENT_WRITE_EOP - flush caches, send int */
1423         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1424         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1425         radeon_ring_write(ring, lower_32_bits(addr));
1426         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1427         radeon_ring_write(ring, fence->seq);
1428         radeon_ring_write(ring, 0);
1429 }
1430
1431 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1432 {
1433         struct radeon_ring *ring = &rdev->ring[ib->ring];
1434         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1435         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1436                 PACKET3_SH_ACTION_ENA;
1437
1438         /* set to DX10/11 mode */
1439         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1440         radeon_ring_write(ring, 1);
1441
1442         if (ring->rptr_save_reg) {
1443                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1444                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1445                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1446                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1447                 radeon_ring_write(ring, next_rptr);
1448         }
1449
1450         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1451         radeon_ring_write(ring,
1452 #ifdef __BIG_ENDIAN
1453                           (2 << 0) |
1454 #endif
1455                           (ib->gpu_addr & 0xFFFFFFFC));
1456         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1457         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1458
1459         /* flush read cache over gart for this vmid */
1460         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1461         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1462         radeon_ring_write(ring, 0xFFFFFFFF);
1463         radeon_ring_write(ring, 0);
1464         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1465 }
1466
1467 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1468 {
1469         if (enable)
1470                 WREG32(CP_ME_CNTL, 0);
1471         else {
1472                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1473                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1474                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1475                 WREG32(SCRATCH_UMSK, 0);
1476                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1477         }
1478 }
1479
1480 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1481                         struct radeon_ring *ring)
1482 {
1483         u32 rptr;
1484
1485         if (rdev->wb.enabled)
1486                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1487         else {
1488                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1489                         rptr = RREG32(CP_RB0_RPTR);
1490                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1491                         rptr = RREG32(CP_RB1_RPTR);
1492                 else
1493                         rptr = RREG32(CP_RB2_RPTR);
1494         }
1495
1496         return rptr;
1497 }
1498
1499 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1500                         struct radeon_ring *ring)
1501 {
1502         u32 wptr;
1503
1504         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1505                 wptr = RREG32(CP_RB0_WPTR);
1506         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1507                 wptr = RREG32(CP_RB1_WPTR);
1508         else
1509                 wptr = RREG32(CP_RB2_WPTR);
1510
1511         return wptr;
1512 }
1513
1514 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1515                          struct radeon_ring *ring)
1516 {
1517         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1518                 WREG32(CP_RB0_WPTR, ring->wptr);
1519                 (void)RREG32(CP_RB0_WPTR);
1520         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1521                 WREG32(CP_RB1_WPTR, ring->wptr);
1522                 (void)RREG32(CP_RB1_WPTR);
1523         } else {
1524                 WREG32(CP_RB2_WPTR, ring->wptr);
1525                 (void)RREG32(CP_RB2_WPTR);
1526         }
1527 }
1528
1529 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1530 {
1531         const __be32 *fw_data;
1532         int i;
1533
1534         if (!rdev->me_fw || !rdev->pfp_fw)
1535                 return -EINVAL;
1536
1537         cayman_cp_enable(rdev, false);
1538
1539         fw_data = (const __be32 *)rdev->pfp_fw->data;
1540         WREG32(CP_PFP_UCODE_ADDR, 0);
1541         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1542                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1543         WREG32(CP_PFP_UCODE_ADDR, 0);
1544
1545         fw_data = (const __be32 *)rdev->me_fw->data;
1546         WREG32(CP_ME_RAM_WADDR, 0);
1547         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1548                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1549
1550         WREG32(CP_PFP_UCODE_ADDR, 0);
1551         WREG32(CP_ME_RAM_WADDR, 0);
1552         WREG32(CP_ME_RAM_RADDR, 0);
1553         return 0;
1554 }
1555
1556 static int cayman_cp_start(struct radeon_device *rdev)
1557 {
1558         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1559         int r, i;
1560
1561         r = radeon_ring_lock(rdev, ring, 7);
1562         if (r) {
1563                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1564                 return r;
1565         }
1566         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1567         radeon_ring_write(ring, 0x1);
1568         radeon_ring_write(ring, 0x0);
1569         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1570         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1571         radeon_ring_write(ring, 0);
1572         radeon_ring_write(ring, 0);
1573         radeon_ring_unlock_commit(rdev, ring, false);
1574
1575         cayman_cp_enable(rdev, true);
1576
1577         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1578         if (r) {
1579                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1580                 return r;
1581         }
1582
1583         /* setup clear context state */
1584         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1585         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1586
1587         for (i = 0; i < cayman_default_size; i++)
1588                 radeon_ring_write(ring, cayman_default_state[i]);
1589
1590         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1591         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1592
1593         /* set clear context state */
1594         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1595         radeon_ring_write(ring, 0);
1596
1597         /* SQ_VTX_BASE_VTX_LOC */
1598         radeon_ring_write(ring, 0xc0026f00);
1599         radeon_ring_write(ring, 0x00000000);
1600         radeon_ring_write(ring, 0x00000000);
1601         radeon_ring_write(ring, 0x00000000);
1602
1603         /* Clear consts */
1604         radeon_ring_write(ring, 0xc0036f00);
1605         radeon_ring_write(ring, 0x00000bc4);
1606         radeon_ring_write(ring, 0xffffffff);
1607         radeon_ring_write(ring, 0xffffffff);
1608         radeon_ring_write(ring, 0xffffffff);
1609
1610         radeon_ring_write(ring, 0xc0026900);
1611         radeon_ring_write(ring, 0x00000316);
1612         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1613         radeon_ring_write(ring, 0x00000010); /*  */
1614
1615         radeon_ring_unlock_commit(rdev, ring, false);
1616
1617         /* XXX init other rings */
1618
1619         return 0;
1620 }
1621
1622 static void cayman_cp_fini(struct radeon_device *rdev)
1623 {
1624         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1625         cayman_cp_enable(rdev, false);
1626         radeon_ring_fini(rdev, ring);
1627         radeon_scratch_free(rdev, ring->rptr_save_reg);
1628 }
1629
1630 static int cayman_cp_resume(struct radeon_device *rdev)
1631 {
1632         static const int ridx[] = {
1633                 RADEON_RING_TYPE_GFX_INDEX,
1634                 CAYMAN_RING_TYPE_CP1_INDEX,
1635                 CAYMAN_RING_TYPE_CP2_INDEX
1636         };
1637         static const unsigned cp_rb_cntl[] = {
1638                 CP_RB0_CNTL,
1639                 CP_RB1_CNTL,
1640                 CP_RB2_CNTL,
1641         };
1642         static const unsigned cp_rb_rptr_addr[] = {
1643                 CP_RB0_RPTR_ADDR,
1644                 CP_RB1_RPTR_ADDR,
1645                 CP_RB2_RPTR_ADDR
1646         };
1647         static const unsigned cp_rb_rptr_addr_hi[] = {
1648                 CP_RB0_RPTR_ADDR_HI,
1649                 CP_RB1_RPTR_ADDR_HI,
1650                 CP_RB2_RPTR_ADDR_HI
1651         };
1652         static const unsigned cp_rb_base[] = {
1653                 CP_RB0_BASE,
1654                 CP_RB1_BASE,
1655                 CP_RB2_BASE
1656         };
1657         static const unsigned cp_rb_rptr[] = {
1658                 CP_RB0_RPTR,
1659                 CP_RB1_RPTR,
1660                 CP_RB2_RPTR
1661         };
1662         static const unsigned cp_rb_wptr[] = {
1663                 CP_RB0_WPTR,
1664                 CP_RB1_WPTR,
1665                 CP_RB2_WPTR
1666         };
1667         struct radeon_ring *ring;
1668         int i, r;
1669
1670         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1671         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1672                                  SOFT_RESET_PA |
1673                                  SOFT_RESET_SH |
1674                                  SOFT_RESET_VGT |
1675                                  SOFT_RESET_SPI |
1676                                  SOFT_RESET_SX));
1677         RREG32(GRBM_SOFT_RESET);
1678         mdelay(15);
1679         WREG32(GRBM_SOFT_RESET, 0);
1680         RREG32(GRBM_SOFT_RESET);
1681
1682         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1683         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1684
1685         /* Set the write pointer delay */
1686         WREG32(CP_RB_WPTR_DELAY, 0);
1687
1688         WREG32(CP_DEBUG, (1 << 27));
1689
1690         /* set the wb address whether it's enabled or not */
1691         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1692         WREG32(SCRATCH_UMSK, 0xff);
1693
1694         for (i = 0; i < 3; ++i) {
1695                 uint32_t rb_cntl;
1696                 uint64_t addr;
1697
1698                 /* Set ring buffer size */
1699                 ring = &rdev->ring[ridx[i]];
1700                 rb_cntl = order_base_2(ring->ring_size / 8);
1701                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1702 #ifdef __BIG_ENDIAN
1703                 rb_cntl |= BUF_SWAP_32BIT;
1704 #endif
1705                 WREG32(cp_rb_cntl[i], rb_cntl);
1706
1707                 /* set the wb address whether it's enabled or not */
1708                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1709                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1710                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1711         }
1712
1713         /* set the rb base addr, this causes an internal reset of ALL rings */
1714         for (i = 0; i < 3; ++i) {
1715                 ring = &rdev->ring[ridx[i]];
1716                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1717         }
1718
1719         for (i = 0; i < 3; ++i) {
1720                 /* Initialize the ring buffer's read and write pointers */
1721                 ring = &rdev->ring[ridx[i]];
1722                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1723
1724                 ring->wptr = 0;
1725                 WREG32(cp_rb_rptr[i], 0);
1726                 WREG32(cp_rb_wptr[i], ring->wptr);
1727
1728                 mdelay(1);
1729                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1730         }
1731
1732         /* start the rings */
1733         cayman_cp_start(rdev);
1734         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1735         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1736         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1737         /* this only test cp0 */
1738         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1739         if (r) {
1740                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1741                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1742                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1743                 return r;
1744         }
1745
1746         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1747                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1748
1749         return 0;
1750 }
1751
1752 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1753 {
1754         u32 reset_mask = 0;
1755         u32 tmp;
1756
1757         /* GRBM_STATUS */
1758         tmp = RREG32(GRBM_STATUS);
1759         if (tmp & (PA_BUSY | SC_BUSY |
1760                    SH_BUSY | SX_BUSY |
1761                    TA_BUSY | VGT_BUSY |
1762                    DB_BUSY | CB_BUSY |
1763                    GDS_BUSY | SPI_BUSY |
1764                    IA_BUSY | IA_BUSY_NO_DMA))
1765                 reset_mask |= RADEON_RESET_GFX;
1766
1767         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1768                    CP_BUSY | CP_COHERENCY_BUSY))
1769                 reset_mask |= RADEON_RESET_CP;
1770
1771         if (tmp & GRBM_EE_BUSY)
1772                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1773
1774         /* DMA_STATUS_REG 0 */
1775         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1776         if (!(tmp & DMA_IDLE))
1777                 reset_mask |= RADEON_RESET_DMA;
1778
1779         /* DMA_STATUS_REG 1 */
1780         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1781         if (!(tmp & DMA_IDLE))
1782                 reset_mask |= RADEON_RESET_DMA1;
1783
1784         /* SRBM_STATUS2 */
1785         tmp = RREG32(SRBM_STATUS2);
1786         if (tmp & DMA_BUSY)
1787                 reset_mask |= RADEON_RESET_DMA;
1788
1789         if (tmp & DMA1_BUSY)
1790                 reset_mask |= RADEON_RESET_DMA1;
1791
1792         /* SRBM_STATUS */
1793         tmp = RREG32(SRBM_STATUS);
1794         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1795                 reset_mask |= RADEON_RESET_RLC;
1796
1797         if (tmp & IH_BUSY)
1798                 reset_mask |= RADEON_RESET_IH;
1799
1800         if (tmp & SEM_BUSY)
1801                 reset_mask |= RADEON_RESET_SEM;
1802
1803         if (tmp & GRBM_RQ_PENDING)
1804                 reset_mask |= RADEON_RESET_GRBM;
1805
1806         if (tmp & VMC_BUSY)
1807                 reset_mask |= RADEON_RESET_VMC;
1808
1809         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1810                    MCC_BUSY | MCD_BUSY))
1811                 reset_mask |= RADEON_RESET_MC;
1812
1813         if (evergreen_is_display_hung(rdev))
1814                 reset_mask |= RADEON_RESET_DISPLAY;
1815
1816         /* VM_L2_STATUS */
1817         tmp = RREG32(VM_L2_STATUS);
1818         if (tmp & L2_BUSY)
1819                 reset_mask |= RADEON_RESET_VMC;
1820
1821         /* Skip MC reset as it's mostly likely not hung, just busy */
1822         if (reset_mask & RADEON_RESET_MC) {
1823                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1824                 reset_mask &= ~RADEON_RESET_MC;
1825         }
1826
1827         return reset_mask;
1828 }
1829
1830 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1831 {
1832         struct evergreen_mc_save save;
1833         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1834         u32 tmp;
1835
1836         if (reset_mask == 0)
1837                 return;
1838
1839         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1840
1841         evergreen_print_gpu_status_regs(rdev);
1842         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1843                  RREG32(0x14F8));
1844         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1845                  RREG32(0x14D8));
1846         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1847                  RREG32(0x14FC));
1848         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1849                  RREG32(0x14DC));
1850
1851         /* Disable CP parsing/prefetching */
1852         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1853
1854         if (reset_mask & RADEON_RESET_DMA) {
1855                 /* dma0 */
1856                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1857                 tmp &= ~DMA_RB_ENABLE;
1858                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1859         }
1860
1861         if (reset_mask & RADEON_RESET_DMA1) {
1862                 /* dma1 */
1863                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1864                 tmp &= ~DMA_RB_ENABLE;
1865                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1866         }
1867
1868         udelay(50);
1869
1870         evergreen_mc_stop(rdev, &save);
1871         if (evergreen_mc_wait_for_idle(rdev)) {
1872                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1873         }
1874
1875         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1876                 grbm_soft_reset = SOFT_RESET_CB |
1877                         SOFT_RESET_DB |
1878                         SOFT_RESET_GDS |
1879                         SOFT_RESET_PA |
1880                         SOFT_RESET_SC |
1881                         SOFT_RESET_SPI |
1882                         SOFT_RESET_SH |
1883                         SOFT_RESET_SX |
1884                         SOFT_RESET_TC |
1885                         SOFT_RESET_TA |
1886                         SOFT_RESET_VGT |
1887                         SOFT_RESET_IA;
1888         }
1889
1890         if (reset_mask & RADEON_RESET_CP) {
1891                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1892
1893                 srbm_soft_reset |= SOFT_RESET_GRBM;
1894         }
1895
1896         if (reset_mask & RADEON_RESET_DMA)
1897                 srbm_soft_reset |= SOFT_RESET_DMA;
1898
1899         if (reset_mask & RADEON_RESET_DMA1)
1900                 srbm_soft_reset |= SOFT_RESET_DMA1;
1901
1902         if (reset_mask & RADEON_RESET_DISPLAY)
1903                 srbm_soft_reset |= SOFT_RESET_DC;
1904
1905         if (reset_mask & RADEON_RESET_RLC)
1906                 srbm_soft_reset |= SOFT_RESET_RLC;
1907
1908         if (reset_mask & RADEON_RESET_SEM)
1909                 srbm_soft_reset |= SOFT_RESET_SEM;
1910
1911         if (reset_mask & RADEON_RESET_IH)
1912                 srbm_soft_reset |= SOFT_RESET_IH;
1913
1914         if (reset_mask & RADEON_RESET_GRBM)
1915                 srbm_soft_reset |= SOFT_RESET_GRBM;
1916
1917         if (reset_mask & RADEON_RESET_VMC)
1918                 srbm_soft_reset |= SOFT_RESET_VMC;
1919
1920         if (!(rdev->flags & RADEON_IS_IGP)) {
1921                 if (reset_mask & RADEON_RESET_MC)
1922                         srbm_soft_reset |= SOFT_RESET_MC;
1923         }
1924
1925         if (grbm_soft_reset) {
1926                 tmp = RREG32(GRBM_SOFT_RESET);
1927                 tmp |= grbm_soft_reset;
1928                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1929                 WREG32(GRBM_SOFT_RESET, tmp);
1930                 tmp = RREG32(GRBM_SOFT_RESET);
1931
1932                 udelay(50);
1933
1934                 tmp &= ~grbm_soft_reset;
1935                 WREG32(GRBM_SOFT_RESET, tmp);
1936                 tmp = RREG32(GRBM_SOFT_RESET);
1937         }
1938
1939         if (srbm_soft_reset) {
1940                 tmp = RREG32(SRBM_SOFT_RESET);
1941                 tmp |= srbm_soft_reset;
1942                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1943                 WREG32(SRBM_SOFT_RESET, tmp);
1944                 tmp = RREG32(SRBM_SOFT_RESET);
1945
1946                 udelay(50);
1947
1948                 tmp &= ~srbm_soft_reset;
1949                 WREG32(SRBM_SOFT_RESET, tmp);
1950                 tmp = RREG32(SRBM_SOFT_RESET);
1951         }
1952
1953         /* Wait a little for things to settle down */
1954         udelay(50);
1955
1956         evergreen_mc_resume(rdev, &save);
1957         udelay(50);
1958
1959         evergreen_print_gpu_status_regs(rdev);
1960 }
1961
1962 int cayman_asic_reset(struct radeon_device *rdev)
1963 {
1964         u32 reset_mask;
1965
1966         reset_mask = cayman_gpu_check_soft_reset(rdev);
1967
1968         if (reset_mask)
1969                 r600_set_bios_scratch_engine_hung(rdev, true);
1970
1971         cayman_gpu_soft_reset(rdev, reset_mask);
1972
1973         reset_mask = cayman_gpu_check_soft_reset(rdev);
1974
1975         if (reset_mask)
1976                 evergreen_gpu_pci_config_reset(rdev);
1977
1978         r600_set_bios_scratch_engine_hung(rdev, false);
1979
1980         return 0;
1981 }
1982
1983 /**
1984  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1985  *
1986  * @rdev: radeon_device pointer
1987  * @ring: radeon_ring structure holding ring information
1988  *
1989  * Check if the GFX engine is locked up.
1990  * Returns true if the engine appears to be locked up, false if not.
1991  */
1992 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1993 {
1994         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1995
1996         if (!(reset_mask & (RADEON_RESET_GFX |
1997                             RADEON_RESET_COMPUTE |
1998                             RADEON_RESET_CP))) {
1999                 radeon_ring_lockup_update(rdev, ring);
2000                 return false;
2001         }
2002         return radeon_ring_test_lockup(rdev, ring);
2003 }
2004
2005 static int cayman_startup(struct radeon_device *rdev)
2006 {
2007         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2008         int r;
2009
2010         /* enable pcie gen2 link */
2011         evergreen_pcie_gen2_enable(rdev);
2012         /* enable aspm */
2013         evergreen_program_aspm(rdev);
2014
2015         /* scratch needs to be initialized before MC */
2016         r = r600_vram_scratch_init(rdev);
2017         if (r)
2018                 return r;
2019
2020         evergreen_mc_program(rdev);
2021
2022         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2023                 r = ni_mc_load_microcode(rdev);
2024                 if (r) {
2025                         DRM_ERROR("Failed to load MC firmware!\n");
2026                         return r;
2027                 }
2028         }
2029
2030         r = cayman_pcie_gart_enable(rdev);
2031         if (r)
2032                 return r;
2033         cayman_gpu_init(rdev);
2034
2035         /* allocate rlc buffers */
2036         if (rdev->flags & RADEON_IS_IGP) {
2037                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2038                 rdev->rlc.reg_list_size =
2039                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2040                 rdev->rlc.cs_data = cayman_cs_data;
2041                 r = sumo_rlc_init(rdev);
2042                 if (r) {
2043                         DRM_ERROR("Failed to init rlc BOs!\n");
2044                         return r;
2045                 }
2046         }
2047
2048         /* allocate wb buffer */
2049         r = radeon_wb_init(rdev);
2050         if (r)
2051                 return r;
2052
2053         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2054         if (r) {
2055                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2056                 return r;
2057         }
2058
2059         r = uvd_v2_2_resume(rdev);
2060         if (!r) {
2061                 r = radeon_fence_driver_start_ring(rdev,
2062                                                    R600_RING_TYPE_UVD_INDEX);
2063                 if (r)
2064                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2065         }
2066         if (r)
2067                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2068
2069         if (rdev->family == CHIP_ARUBA) {
2070                 r = radeon_vce_resume(rdev);
2071                 if (!r)
2072                         r = vce_v1_0_resume(rdev);
2073
2074                 if (!r)
2075                         r = radeon_fence_driver_start_ring(rdev,
2076                                                            TN_RING_TYPE_VCE1_INDEX);
2077                 if (!r)
2078                         r = radeon_fence_driver_start_ring(rdev,
2079                                                            TN_RING_TYPE_VCE2_INDEX);
2080
2081                 if (r) {
2082                         dev_err(rdev->dev, "VCE init error (%d).\n", r);
2083                         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2084                         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2085                 }
2086         }
2087
2088         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2089         if (r) {
2090                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2091                 return r;
2092         }
2093
2094         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2095         if (r) {
2096                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2097                 return r;
2098         }
2099
2100         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2101         if (r) {
2102                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2103                 return r;
2104         }
2105
2106         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2107         if (r) {
2108                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2109                 return r;
2110         }
2111
2112         /* Enable IRQ */
2113         if (!rdev->irq.installed) {
2114                 r = radeon_irq_kms_init(rdev);
2115                 if (r)
2116                         return r;
2117         }
2118
2119         r = r600_irq_init(rdev);
2120         if (r) {
2121                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2122                 radeon_irq_kms_fini(rdev);
2123                 return r;
2124         }
2125         evergreen_irq_set(rdev);
2126
2127         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2128                              RADEON_CP_PACKET2);
2129         if (r)
2130                 return r;
2131
2132         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2133         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2134                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2135         if (r)
2136                 return r;
2137
2138         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2139         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2140                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2141         if (r)
2142                 return r;
2143
2144         r = cayman_cp_load_microcode(rdev);
2145         if (r)
2146                 return r;
2147         r = cayman_cp_resume(rdev);
2148         if (r)
2149                 return r;
2150
2151         r = cayman_dma_resume(rdev);
2152         if (r)
2153                 return r;
2154
2155         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2156         if (ring->ring_size) {
2157                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2158                                      RADEON_CP_PACKET2);
2159                 if (!r)
2160                         r = uvd_v1_0_init(rdev);
2161                 if (r)
2162                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2163         }
2164
2165         if (rdev->family == CHIP_ARUBA) {
2166                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2167                 if (ring->ring_size)
2168                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2169
2170                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2171                 if (ring->ring_size)
2172                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2173
2174                 if (!r)
2175                         r = vce_v1_0_init(rdev);
2176                 if (r)
2177                         DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
2178         }
2179
2180         r = radeon_ib_pool_init(rdev);
2181         if (r) {
2182                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2183                 return r;
2184         }
2185
2186         r = radeon_vm_manager_init(rdev);
2187         if (r) {
2188                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2189                 return r;
2190         }
2191
2192         r = radeon_audio_init(rdev);
2193         if (r)
2194                 return r;
2195
2196         return 0;
2197 }
2198
2199 int cayman_resume(struct radeon_device *rdev)
2200 {
2201         int r;
2202
2203         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2204          * posting will perform necessary task to bring back GPU into good
2205          * shape.
2206          */
2207         /* post card */
2208         atom_asic_init(rdev->mode_info.atom_context);
2209
2210         /* init golden registers */
2211         ni_init_golden_registers(rdev);
2212
2213         if (rdev->pm.pm_method == PM_METHOD_DPM)
2214                 radeon_pm_resume(rdev);
2215
2216         rdev->accel_working = true;
2217         r = cayman_startup(rdev);
2218         if (r) {
2219                 DRM_ERROR("cayman startup failed on resume\n");
2220                 rdev->accel_working = false;
2221                 return r;
2222         }
2223         return r;
2224 }
2225
2226 int cayman_suspend(struct radeon_device *rdev)
2227 {
2228         radeon_pm_suspend(rdev);
2229         radeon_audio_fini(rdev);
2230         radeon_vm_manager_fini(rdev);
2231         cayman_cp_enable(rdev, false);
2232         cayman_dma_stop(rdev);
2233         uvd_v1_0_fini(rdev);
2234         radeon_uvd_suspend(rdev);
2235         evergreen_irq_suspend(rdev);
2236         radeon_wb_disable(rdev);
2237         cayman_pcie_gart_disable(rdev);
2238         return 0;
2239 }
2240
2241 /* Plan is to move initialization in that function and use
2242  * helper function so that radeon_device_init pretty much
2243  * do nothing more than calling asic specific function. This
2244  * should also allow to remove a bunch of callback function
2245  * like vram_info.
2246  */
2247 int cayman_init(struct radeon_device *rdev)
2248 {
2249         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2250         int r;
2251
2252         /* Read BIOS */
2253         if (!radeon_get_bios(rdev)) {
2254                 if (ASIC_IS_AVIVO(rdev))
2255                         return -EINVAL;
2256         }
2257         /* Must be an ATOMBIOS */
2258         if (!rdev->is_atom_bios) {
2259                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2260                 return -EINVAL;
2261         }
2262         r = radeon_atombios_init(rdev);
2263         if (r)
2264                 return r;
2265
2266         /* Post card if necessary */
2267         if (!radeon_card_posted(rdev)) {
2268                 if (!rdev->bios) {
2269                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2270                         return -EINVAL;
2271                 }
2272                 DRM_INFO("GPU not posted. posting now...\n");
2273                 atom_asic_init(rdev->mode_info.atom_context);
2274         }
2275         /* init golden registers */
2276         ni_init_golden_registers(rdev);
2277         /* Initialize scratch registers */
2278         r600_scratch_init(rdev);
2279         /* Initialize surface registers */
2280         radeon_surface_init(rdev);
2281         /* Initialize clocks */
2282         radeon_get_clock_info(rdev->ddev);
2283         /* Fence driver */
2284         r = radeon_fence_driver_init(rdev);
2285         if (r)
2286                 return r;
2287         /* initialize memory controller */
2288         r = evergreen_mc_init(rdev);
2289         if (r)
2290                 return r;
2291         /* Memory manager */
2292         r = radeon_bo_init(rdev);
2293         if (r)
2294                 return r;
2295
2296         if (rdev->flags & RADEON_IS_IGP) {
2297                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2298                         r = ni_init_microcode(rdev);
2299                         if (r) {
2300                                 DRM_ERROR("Failed to load firmware!\n");
2301                                 return r;
2302                         }
2303                 }
2304         } else {
2305                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2306                         r = ni_init_microcode(rdev);
2307                         if (r) {
2308                                 DRM_ERROR("Failed to load firmware!\n");
2309                                 return r;
2310                         }
2311                 }
2312         }
2313
2314         /* Initialize power management */
2315         radeon_pm_init(rdev);
2316
2317         ring->ring_obj = NULL;
2318         r600_ring_init(rdev, ring, 1024 * 1024);
2319
2320         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2321         ring->ring_obj = NULL;
2322         r600_ring_init(rdev, ring, 64 * 1024);
2323
2324         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2325         ring->ring_obj = NULL;
2326         r600_ring_init(rdev, ring, 64 * 1024);
2327
2328         r = radeon_uvd_init(rdev);
2329         if (!r) {
2330                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2331                 ring->ring_obj = NULL;
2332                 r600_ring_init(rdev, ring, 4096);
2333         }
2334
2335         if (rdev->family == CHIP_ARUBA) {
2336                 r = radeon_vce_init(rdev);
2337                 if (!r) {
2338                         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2339                         ring->ring_obj = NULL;
2340                         r600_ring_init(rdev, ring, 4096);
2341
2342                         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2343                         ring->ring_obj = NULL;
2344                         r600_ring_init(rdev, ring, 4096);
2345                 }
2346         }
2347
2348         rdev->ih.ring_obj = NULL;
2349         r600_ih_ring_init(rdev, 64 * 1024);
2350
2351         r = r600_pcie_gart_init(rdev);
2352         if (r)
2353                 return r;
2354
2355         rdev->accel_working = true;
2356         r = cayman_startup(rdev);
2357         if (r) {
2358                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2359                 cayman_cp_fini(rdev);
2360                 cayman_dma_fini(rdev);
2361                 r600_irq_fini(rdev);
2362                 if (rdev->flags & RADEON_IS_IGP)
2363                         sumo_rlc_fini(rdev);
2364                 radeon_wb_fini(rdev);
2365                 radeon_ib_pool_fini(rdev);
2366                 radeon_vm_manager_fini(rdev);
2367                 radeon_irq_kms_fini(rdev);
2368                 cayman_pcie_gart_fini(rdev);
2369                 rdev->accel_working = false;
2370         }
2371
2372         /* Don't start up if the MC ucode is missing.
2373          * The default clocks and voltages before the MC ucode
2374          * is loaded are not suffient for advanced operations.
2375          *
2376          * We can skip this check for TN, because there is no MC
2377          * ucode.
2378          */
2379         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2380                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2381                 return -EINVAL;
2382         }
2383
2384         return 0;
2385 }
2386
2387 void cayman_fini(struct radeon_device *rdev)
2388 {
2389         radeon_pm_fini(rdev);
2390         cayman_cp_fini(rdev);
2391         cayman_dma_fini(rdev);
2392         r600_irq_fini(rdev);
2393         if (rdev->flags & RADEON_IS_IGP)
2394                 sumo_rlc_fini(rdev);
2395         radeon_wb_fini(rdev);
2396         radeon_vm_manager_fini(rdev);
2397         radeon_ib_pool_fini(rdev);
2398         radeon_irq_kms_fini(rdev);
2399         uvd_v1_0_fini(rdev);
2400         radeon_uvd_fini(rdev);
2401         if (rdev->family == CHIP_ARUBA)
2402                 radeon_vce_fini(rdev);
2403         cayman_pcie_gart_fini(rdev);
2404         r600_vram_scratch_fini(rdev);
2405         radeon_gem_fini(rdev);
2406         radeon_fence_driver_fini(rdev);
2407         radeon_bo_fini(rdev);
2408         radeon_atombios_fini(rdev);
2409         kfree(rdev->bios);
2410         rdev->bios = NULL;
2411 }
2412
2413 /*
2414  * vm
2415  */
2416 int cayman_vm_init(struct radeon_device *rdev)
2417 {
2418         /* number of VMs */
2419         rdev->vm_manager.nvm = 8;
2420         /* base offset of vram pages */
2421         if (rdev->flags & RADEON_IS_IGP) {
2422                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2423                 tmp <<= 22;
2424                 rdev->vm_manager.vram_base_offset = tmp;
2425         } else
2426                 rdev->vm_manager.vram_base_offset = 0;
2427         return 0;
2428 }
2429
2430 void cayman_vm_fini(struct radeon_device *rdev)
2431 {
2432 }
2433
2434 /**
2435  * cayman_vm_decode_fault - print human readable fault info
2436  *
2437  * @rdev: radeon_device pointer
2438  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2439  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2440  *
2441  * Print human readable fault information (cayman/TN).
2442  */
2443 void cayman_vm_decode_fault(struct radeon_device *rdev,
2444                             u32 status, u32 addr)
2445 {
2446         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2447         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2448         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2449         char *block;
2450
2451         switch (mc_id) {
2452         case 32:
2453         case 16:
2454         case 96:
2455         case 80:
2456         case 160:
2457         case 144:
2458         case 224:
2459         case 208:
2460                 block = "CB";
2461                 break;
2462         case 33:
2463         case 17:
2464         case 97:
2465         case 81:
2466         case 161:
2467         case 145:
2468         case 225:
2469         case 209:
2470                 block = "CB_FMASK";
2471                 break;
2472         case 34:
2473         case 18:
2474         case 98:
2475         case 82:
2476         case 162:
2477         case 146:
2478         case 226:
2479         case 210:
2480                 block = "CB_CMASK";
2481                 break;
2482         case 35:
2483         case 19:
2484         case 99:
2485         case 83:
2486         case 163:
2487         case 147:
2488         case 227:
2489         case 211:
2490                 block = "CB_IMMED";
2491                 break;
2492         case 36:
2493         case 20:
2494         case 100:
2495         case 84:
2496         case 164:
2497         case 148:
2498         case 228:
2499         case 212:
2500                 block = "DB";
2501                 break;
2502         case 37:
2503         case 21:
2504         case 101:
2505         case 85:
2506         case 165:
2507         case 149:
2508         case 229:
2509         case 213:
2510                 block = "DB_HTILE";
2511                 break;
2512         case 38:
2513         case 22:
2514         case 102:
2515         case 86:
2516         case 166:
2517         case 150:
2518         case 230:
2519         case 214:
2520                 block = "SX";
2521                 break;
2522         case 39:
2523         case 23:
2524         case 103:
2525         case 87:
2526         case 167:
2527         case 151:
2528         case 231:
2529         case 215:
2530                 block = "DB_STEN";
2531                 break;
2532         case 40:
2533         case 24:
2534         case 104:
2535         case 88:
2536         case 232:
2537         case 216:
2538         case 168:
2539         case 152:
2540                 block = "TC_TFETCH";
2541                 break;
2542         case 41:
2543         case 25:
2544         case 105:
2545         case 89:
2546         case 233:
2547         case 217:
2548         case 169:
2549         case 153:
2550                 block = "TC_VFETCH";
2551                 break;
2552         case 42:
2553         case 26:
2554         case 106:
2555         case 90:
2556         case 234:
2557         case 218:
2558         case 170:
2559         case 154:
2560                 block = "VC";
2561                 break;
2562         case 112:
2563                 block = "CP";
2564                 break;
2565         case 113:
2566         case 114:
2567                 block = "SH";
2568                 break;
2569         case 115:
2570                 block = "VGT";
2571                 break;
2572         case 178:
2573                 block = "IH";
2574                 break;
2575         case 51:
2576                 block = "RLC";
2577                 break;
2578         case 55:
2579                 block = "DMA";
2580                 break;
2581         case 56:
2582                 block = "HDP";
2583                 break;
2584         default:
2585                 block = "unknown";
2586                 break;
2587         }
2588
2589         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2590                protections, vmid, addr,
2591                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2592                block, mc_id);
2593 }
2594
2595 /**
2596  * cayman_vm_flush - vm flush using the CP
2597  *
2598  * @rdev: radeon_device pointer
2599  *
2600  * Update the page table base and flush the VM TLB
2601  * using the CP (cayman-si).
2602  */
2603 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2604                      unsigned vm_id, uint64_t pd_addr)
2605 {
2606         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2607         radeon_ring_write(ring, pd_addr >> 12);
2608
2609         /* flush hdp cache */
2610         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2611         radeon_ring_write(ring, 0x1);
2612
2613         /* bits 0-7 are the VM contexts0-7 */
2614         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2615         radeon_ring_write(ring, 1 << vm_id);
2616
2617         /* wait for the invalidate to complete */
2618         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2619         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2620                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2621         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2622         radeon_ring_write(ring, 0);
2623         radeon_ring_write(ring, 0); /* ref */
2624         radeon_ring_write(ring, 0); /* mask */
2625         radeon_ring_write(ring, 0x20); /* poll interval */
2626
2627         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2628         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2629         radeon_ring_write(ring, 0x0);
2630 }
2631
2632 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2633 {
2634         struct atom_clock_dividers dividers;
2635         int r, i;
2636
2637         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2638                                            ecclk, false, &dividers);
2639         if (r)
2640                 return r;
2641
2642         for (i = 0; i < 100; i++) {
2643                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2644                         break;
2645                 mdelay(10);
2646         }
2647         if (i == 100)
2648                 return -ETIMEDOUT;
2649
2650         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2651
2652         for (i = 0; i < 100; i++) {
2653                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2654                         break;
2655                 mdelay(10);
2656         }
2657         if (i == 100)
2658                 return -ETIMEDOUT;
2659
2660         return 0;
2661 }