drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 static const u32 tn_rlc_save_restore_register_list[] =
  40 {
  41         0x98fc,
  42         0x98f0,
  43         0x9834,
  44         0x9838,
  45         0x9870,
  46         0x9874,
  47         0x8a14,
  48         0x8b24,
  49         0x8bcc,
  50         0x8b10,
  51         0x8c30,
  52         0x8d00,
  53         0x8d04,
  54         0x8c00,
  55         0x8c04,
  56         0x8c10,
  57         0x8c14,
  58         0x8d8c,
  59         0x8cf0,
  60         0x8e38,
  61         0x9508,
  62         0x9688,
  63         0x9608,
  64         0x960c,
  65         0x9610,
  66         0x9614,
  67         0x88c4,
  68         0x8978,
  69         0x88d4,
  70         0x900c,
  71         0x9100,
  72         0x913c,
  73         0x90e8,
  74         0x9354,
  75         0xa008,
  76         0x98f8,
  77         0x9148,
  78         0x914c,
  79         0x3f94,
  80         0x98f4,
  81         0x9b7c,
  82         0x3f8c,
  83         0x8950,
  84         0x8954,
  85         0x8a18,
  86         0x8b28,
  87         0x9144,
  88         0x3f90,
  89         0x915c,
  90         0x9160,
  91         0x9178,
  92         0x917c,
  93         0x9180,
  94         0x918c,
  95         0x9190,
  96         0x9194,
  97         0x9198,
  98         0x919c,
  99         0x91a8,
 100         0x91ac,
 101         0x91b0,
 102         0x91b4,
 103         0x91b8,
 104         0x91c4,
 105         0x91c8,
 106         0x91cc,
 107         0x91d0,
 108         0x91d4,
 109         0x91e0,
 110         0x91e4,
 111         0x91ec,
 112         0x91f0,
 113         0x91f4,
 114         0x9200,
 115         0x9204,
 116         0x929c,
 117         0x8030,
 118         0x9150,
 119         0x9a60,
 120         0x920c,
 121         0x9210,
 122         0x9228,
 123         0x922c,
 124         0x9244,
 125         0x9248,
 126         0x91e8,
 127         0x9294,
 128         0x9208,
 129         0x9224,
 130         0x9240,
 131         0x9220,
 132         0x923c,
 133         0x9258,
 134         0x9744,
 135         0xa200,
 136         0xa204,
 137         0xa208,
 138         0xa20c,
 139         0x8d58,
 140         0x9030,
 141         0x9034,
 142         0x9038,
 143         0x903c,
 144         0x9040,
 145         0x9654,
 146         0x897c,
 147         0xa210,
 148         0xa214,
 149         0x9868,
 150         0xa02c,
 151         0x9664,
 152         0x9698,
 153         0x949c,
 154         0x8e10,
 155         0x8e18,
 156         0x8c50,
 157         0x8c58,
 158         0x8c60,
 159         0x8c68,
 160         0x89b4,
 161         0x9830,
 162         0x802c,
 163 };
 164
 165 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 166 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 167 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 169 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 170 extern void evergreen_mc_program(struct radeon_device *rdev);
 171 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 172 extern int evergreen_mc_init(struct radeon_device *rdev);
 173 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 174 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 175 extern void evergreen_program_aspm(struct radeon_device *rdev);
 176 extern void sumo_rlc_fini(struct radeon_device *rdev);
 177 extern int sumo_rlc_init(struct radeon_device *rdev);
 178 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 179
 180 /* Firmware Names */
 181 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 184 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 185 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 189 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 193 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 198 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 201 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 202
 203
 204 static const u32 cayman_golden_registers2[] =
 205 {
 206         0x3e5c, 0xffffffff, 0x00000000,
 207         0x3e48, 0xffffffff, 0x00000000,
 208         0x3e4c, 0xffffffff, 0x00000000,
 209         0x3e64, 0xffffffff, 0x00000000,
 210         0x3e50, 0xffffffff, 0x00000000,
 211         0x3e60, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cayman_golden_registers[] =
 215 {
 216         0x5eb4, 0xffffffff, 0x00000002,
 217         0x5e78, 0x8f311ff1, 0x001000f0,
 218         0x3f90, 0xffff0000, 0xff000000,
 219         0x9148, 0xffff0000, 0xff000000,
 220         0x3f94, 0xffff0000, 0xff000000,
 221         0x914c, 0xffff0000, 0xff000000,
 222         0xc78, 0x00000080, 0x00000080,
 223         0xbd4, 0x70073777, 0x00011003,
 224         0xd02c, 0xbfffff1f, 0x08421000,
 225         0xd0b8, 0x73773777, 0x02011003,
 226         0x5bc0, 0x00200000, 0x50100000,
 227         0x98f8, 0x33773777, 0x02011003,
 228         0x98fc, 0xffffffff, 0x76541032,
 229         0x7030, 0x31000311, 0x00000011,
 230         0x2f48, 0x33773777, 0x42010001,
 231         0x6b28, 0x00000010, 0x00000012,
 232         0x7728, 0x00000010, 0x00000012,
 233         0x10328, 0x00000010, 0x00000012,
 234         0x10f28, 0x00000010, 0x00000012,
 235         0x11b28, 0x00000010, 0x00000012,
 236         0x12728, 0x00000010, 0x00000012,
 237         0x240c, 0x000007ff, 0x00000000,
 238         0x8a14, 0xf000001f, 0x00000007,
 239         0x8b24, 0x3fff3fff, 0x00ff0fff,
 240         0x8b10, 0x0000ff0f, 0x00000000,
 241         0x28a4c, 0x07ffffff, 0x06000000,
 242         0x10c, 0x00000001, 0x00010003,
 243         0xa02c, 0xffffffff, 0x0000009b,
 244         0x913c, 0x0000010f, 0x01000100,
 245         0x8c04, 0xf8ff00ff, 0x40600060,
 246         0x28350, 0x00000f01, 0x00000000,
 247         0x9508, 0x3700001f, 0x00000002,
 248         0x960c, 0xffffffff, 0x54763210,
 249         0x88c4, 0x001f3ae3, 0x00000082,
 250         0x88d0, 0xffffffff, 0x0f40df40,
 251         0x88d4, 0x0000001f, 0x00000010,
 252         0x8974, 0xffffffff, 0x00000000
 253 };
 254
 255 static const u32 dvst_golden_registers2[] =
 256 {
 257         0x8f8, 0xffffffff, 0,
 258         0x8fc, 0x00380000, 0,
 259         0x8f8, 0xffffffff, 1,
 260         0x8fc, 0x0e000000, 0
 261 };
 262
 263 static const u32 dvst_golden_registers[] =
 264 {
 265         0x690, 0x3fff3fff, 0x20c00033,
 266         0x918c, 0x0fff0fff, 0x00010006,
 267         0x91a8, 0x0fff0fff, 0x00010006,
 268         0x9150, 0xffffdfff, 0x6e944040,
 269         0x917c, 0x0fff0fff, 0x00030002,
 270         0x9198, 0x0fff0fff, 0x00030002,
 271         0x915c, 0x0fff0fff, 0x00010000,
 272         0x3f90, 0xffff0001, 0xff000000,
 273         0x9178, 0x0fff0fff, 0x00070000,
 274         0x9194, 0x0fff0fff, 0x00070000,
 275         0x9148, 0xffff0001, 0xff000000,
 276         0x9190, 0x0fff0fff, 0x00090008,
 277         0x91ac, 0x0fff0fff, 0x00090008,
 278         0x3f94, 0xffff0000, 0xff000000,
 279         0x914c, 0xffff0000, 0xff000000,
 280         0x929c, 0x00000fff, 0x00000001,
 281         0x55e4, 0xff607fff, 0xfc000100,
 282         0x8a18, 0xff000fff, 0x00000100,
 283         0x8b28, 0xff000fff, 0x00000100,
 284         0x9144, 0xfffc0fff, 0x00000100,
 285         0x6ed8, 0x00010101, 0x00010000,
 286         0x9830, 0xffffffff, 0x00000000,
 287         0x9834, 0xf00fffff, 0x00000400,
 288         0x9838, 0xfffffffe, 0x00000000,
 289         0xd0c0, 0xff000fff, 0x00000100,
 290         0xd02c, 0xbfffff1f, 0x08421000,
 291         0xd0b8, 0x73773777, 0x12010001,
 292         0x5bb0, 0x000000f0, 0x00000070,
 293         0x98f8, 0x73773777, 0x12010001,
 294         0x98fc, 0xffffffff, 0x00000010,
 295         0x9b7c, 0x00ff0000, 0x00fc0000,
 296         0x8030, 0x00001f0f, 0x0000100a,
 297         0x2f48, 0x73773777, 0x12010001,
 298         0x2408, 0x00030000, 0x000c007f,
 299         0x8a14, 0xf000003f, 0x00000007,
 300         0x8b24, 0x3fff3fff, 0x00ff0fff,
 301         0x8b10, 0x0000ff0f, 0x00000000,
 302         0x28a4c, 0x07ffffff, 0x06000000,
 303         0x4d8, 0x00000fff, 0x00000100,
 304         0xa008, 0xffffffff, 0x00010000,
 305         0x913c, 0xffff03ff, 0x01000100,
 306         0x8c00, 0x000000ff, 0x00000003,
 307         0x8c04, 0xf8ff00ff, 0x40600060,
 308         0x8cf0, 0x1fff1fff, 0x08e00410,
 309         0x28350, 0x00000f01, 0x00000000,
 310         0x9508, 0xf700071f, 0x00000002,
 311         0x960c, 0xffffffff, 0x54763210,
 312         0x20ef8, 0x01ff01ff, 0x00000002,
 313         0x20e98, 0xfffffbff, 0x00200000,
 314         0x2015c, 0xffffffff, 0x00000f40,
 315         0x88c4, 0x001f3ae3, 0x00000082,
 316         0x8978, 0x3fffffff, 0x04050140,
 317         0x88d4, 0x0000001f, 0x00000010,
 318         0x8974, 0xffffffff, 0x00000000
 319 };
 320
 321 static const u32 scrapper_golden_registers[] =
 322 {
 323         0x690, 0x3fff3fff, 0x20c00033,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x918c, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x91a8, 0x0fff0fff, 0x00010006,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x9150, 0xffffdfff, 0x6e944040,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x917c, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x9198, 0x0fff0fff, 0x00030002,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x915c, 0x0fff0fff, 0x00010000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x3f90, 0xffff0001, 0xff000000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9178, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9194, 0x0fff0fff, 0x00070000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9148, 0xffff0001, 0xff000000,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x9190, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x91ac, 0x0fff0fff, 0x00090008,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x3f94, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x914c, 0xffff0000, 0xff000000,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x929c, 0x00000fff, 0x00000001,
 354         0x55e4, 0xff607fff, 0xfc000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8a18, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x8b28, 0xff000fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x9144, 0xfffc0fff, 0x00000100,
 361         0x6ed8, 0x00010101, 0x00010000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9830, 0xffffffff, 0x00000000,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9834, 0xf00fffff, 0x00000400,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0x9838, 0xfffffffe, 0x00000000,
 368         0xd0c0, 0xff000fff, 0x00000100,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd02c, 0xbfffff1f, 0x08421000,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0xd0b8, 0x73773777, 0x12010001,
 373         0x5bb0, 0x000000f0, 0x00000070,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98f8, 0x73773777, 0x12010001,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x98fc, 0xffffffff, 0x00000010,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x9b7c, 0x00ff0000, 0x00fc0000,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x8030, 0x00001f0f, 0x0000100a,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2f48, 0x73773777, 0x12010001,
 384         0x2408, 0x00030000, 0x000c007f,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8a14, 0xf000003f, 0x00000007,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b24, 0x3fff3fff, 0x00ff0fff,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x8b10, 0x0000ff0f, 0x00000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x28a4c, 0x07ffffff, 0x06000000,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0x4d8, 0x00000fff, 0x00000100,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0xa008, 0xffffffff, 0x00010000,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x913c, 0xffff03ff, 0x01000100,
 399         0x90e8, 0x001fffff, 0x010400c0,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c00, 0x000000ff, 0x00000003,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c04, 0xf8ff00ff, 0x40600060,
 404         0x8c30, 0x0000000f, 0x00040005,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x8cf0, 0x1fff1fff, 0x08e00410,
 407         0x900c, 0x00ffffff, 0x0017071f,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x28350, 0x00000f01, 0x00000000,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9508, 0xf700071f, 0x00000002,
 412         0x9688, 0x00300000, 0x0017000f,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x960c, 0xffffffff, 0x54763210,
 415         0x20ef8, 0x01ff01ff, 0x00000002,
 416         0x20e98, 0xfffffbff, 0x00200000,
 417         0x2015c, 0xffffffff, 0x00000f40,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x88c4, 0x001f3ae3, 0x00000082,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x8978, 0x3fffffff, 0x04050140,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x88d4, 0x0000001f, 0x00000010,
 424         0x8974, 0xffffffff, 0x00000000,
 425         0x8974, 0xffffffff, 0x00000000
 426 };
 427
 428 static void ni_init_golden_registers(struct radeon_device *rdev)
 429 {
 430         switch (rdev->family) {
 431         case CHIP_CAYMAN:
 432                 radeon_program_register_sequence(rdev,
 433                                                  cayman_golden_registers,
 434                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 435                 radeon_program_register_sequence(rdev,
 436                                                  cayman_golden_registers2,
 437                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 438                 break;
 439         case CHIP_ARUBA:
 440                 if ((rdev->pdev->device == 0x9900) ||
 441                     (rdev->pdev->device == 0x9901) ||
 442                     (rdev->pdev->device == 0x9903) ||
 443                     (rdev->pdev->device == 0x9904) ||
 444                     (rdev->pdev->device == 0x9905) ||
 445                     (rdev->pdev->device == 0x9906) ||
 446                     (rdev->pdev->device == 0x9907) ||
 447                     (rdev->pdev->device == 0x9908) ||
 448                     (rdev->pdev->device == 0x9909) ||
 449                     (rdev->pdev->device == 0x990A) ||
 450                     (rdev->pdev->device == 0x990B) ||
 451                     (rdev->pdev->device == 0x990C) ||
 452                     (rdev->pdev->device == 0x990D) ||
 453                     (rdev->pdev->device == 0x990E) ||
 454                     (rdev->pdev->device == 0x990F) ||
 455                     (rdev->pdev->device == 0x9910) ||
 456                     (rdev->pdev->device == 0x9913) ||
 457                     (rdev->pdev->device == 0x9917) ||
 458                     (rdev->pdev->device == 0x9918)) {
 459                         radeon_program_register_sequence(rdev,
 460                                                          dvst_golden_registers,
 461                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 462                         radeon_program_register_sequence(rdev,
 463                                                          dvst_golden_registers2,
 464                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 465                 } else {
 466                         radeon_program_register_sequence(rdev,
 467                                                          scrapper_golden_registers,
 468                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 469                         radeon_program_register_sequence(rdev,
 470                                                          dvst_golden_registers2,
 471                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 472                 }
 473                 break;
 474         default:
 475                 break;
 476         }
 477 }
 478
 479 #define BTC_IO_MC_REGS_SIZE 29
 480
 481 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 482         {0x00000077, 0xff010100},
 483         {0x00000078, 0x00000000},
 484         {0x00000079, 0x00001434},
 485         {0x0000007a, 0xcc08ec08},
 486         {0x0000007b, 0x00040000},
 487         {0x0000007c, 0x000080c0},
 488         {0x0000007d, 0x09000000},
 489         {0x0000007e, 0x00210404},
 490         {0x00000081, 0x08a8e800},
 491         {0x00000082, 0x00030444},
 492         {0x00000083, 0x00000000},
 493         {0x00000085, 0x00000001},
 494         {0x00000086, 0x00000002},
 495         {0x00000087, 0x48490000},
 496         {0x00000088, 0x20244647},
 497         {0x00000089, 0x00000005},
 498         {0x0000008b, 0x66030000},
 499         {0x0000008c, 0x00006603},
 500         {0x0000008d, 0x00000100},
 501         {0x0000008f, 0x00001c0a},
 502         {0x00000090, 0xff000001},
 503         {0x00000094, 0x00101101},
 504         {0x00000095, 0x00000fff},
 505         {0x00000096, 0x00116fff},
 506         {0x00000097, 0x60010000},
 507         {0x00000098, 0x10010000},
 508         {0x00000099, 0x00006000},
 509         {0x0000009a, 0x00001000},
 510         {0x0000009f, 0x00946a00}
 511 };
 512
 513 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 514         {0x00000077, 0xff010100},
 515         {0x00000078, 0x00000000},
 516         {0x00000079, 0x00001434},
 517         {0x0000007a, 0xcc08ec08},
 518         {0x0000007b, 0x00040000},
 519         {0x0000007c, 0x000080c0},
 520         {0x0000007d, 0x09000000},
 521         {0x0000007e, 0x00210404},
 522         {0x00000081, 0x08a8e800},
 523         {0x00000082, 0x00030444},
 524         {0x00000083, 0x00000000},
 525         {0x00000085, 0x00000001},
 526         {0x00000086, 0x00000002},
 527         {0x00000087, 0x48490000},
 528         {0x00000088, 0x20244647},
 529         {0x00000089, 0x00000005},
 530         {0x0000008b, 0x66030000},
 531         {0x0000008c, 0x00006603},
 532         {0x0000008d, 0x00000100},
 533         {0x0000008f, 0x00001c0a},
 534         {0x00000090, 0xff000001},
 535         {0x00000094, 0x00101101},
 536         {0x00000095, 0x00000fff},
 537         {0x00000096, 0x00116fff},
 538         {0x00000097, 0x60010000},
 539         {0x00000098, 0x10010000},
 540         {0x00000099, 0x00006000},
 541         {0x0000009a, 0x00001000},
 542         {0x0000009f, 0x00936a00}
 543 };
 544
 545 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 546         {0x00000077, 0xff010100},
 547         {0x00000078, 0x00000000},
 548         {0x00000079, 0x00001434},
 549         {0x0000007a, 0xcc08ec08},
 550         {0x0000007b, 0x00040000},
 551         {0x0000007c, 0x000080c0},
 552         {0x0000007d, 0x09000000},
 553         {0x0000007e, 0x00210404},
 554         {0x00000081, 0x08a8e800},
 555         {0x00000082, 0x00030444},
 556         {0x00000083, 0x00000000},
 557         {0x00000085, 0x00000001},
 558         {0x00000086, 0x00000002},
 559         {0x00000087, 0x48490000},
 560         {0x00000088, 0x20244647},
 561         {0x00000089, 0x00000005},
 562         {0x0000008b, 0x66030000},
 563         {0x0000008c, 0x00006603},
 564         {0x0000008d, 0x00000100},
 565         {0x0000008f, 0x00001c0a},
 566         {0x00000090, 0xff000001},
 567         {0x00000094, 0x00101101},
 568         {0x00000095, 0x00000fff},
 569         {0x00000096, 0x00116fff},
 570         {0x00000097, 0x60010000},
 571         {0x00000098, 0x10010000},
 572         {0x00000099, 0x00006000},
 573         {0x0000009a, 0x00001000},
 574         {0x0000009f, 0x00916a00}
 575 };
 576
 577 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 578         {0x00000077, 0xff010100},
 579         {0x00000078, 0x00000000},
 580         {0x00000079, 0x00001434},
 581         {0x0000007a, 0xcc08ec08},
 582         {0x0000007b, 0x00040000},
 583         {0x0000007c, 0x000080c0},
 584         {0x0000007d, 0x09000000},
 585         {0x0000007e, 0x00210404},
 586         {0x00000081, 0x08a8e800},
 587         {0x00000082, 0x00030444},
 588         {0x00000083, 0x00000000},
 589         {0x00000085, 0x00000001},
 590         {0x00000086, 0x00000002},
 591         {0x00000087, 0x48490000},
 592         {0x00000088, 0x20244647},
 593         {0x00000089, 0x00000005},
 594         {0x0000008b, 0x66030000},
 595         {0x0000008c, 0x00006603},
 596         {0x0000008d, 0x00000100},
 597         {0x0000008f, 0x00001c0a},
 598         {0x00000090, 0xff000001},
 599         {0x00000094, 0x00101101},
 600         {0x00000095, 0x00000fff},
 601         {0x00000096, 0x00116fff},
 602         {0x00000097, 0x60010000},
 603         {0x00000098, 0x10010000},
 604         {0x00000099, 0x00006000},
 605         {0x0000009a, 0x00001000},
 606         {0x0000009f, 0x00976b00}
 607 };
 608
 609 int ni_mc_load_microcode(struct radeon_device *rdev)
 610 {
 611         const __be32 *fw_data;
 612         u32 mem_type, running, blackout = 0;
 613         u32 *io_mc_regs;
 614         int i, ucode_size, regs_size;
 615
 616         if (!rdev->mc_fw)
 617                 return -EINVAL;
 618
 619         switch (rdev->family) {
 620         case CHIP_BARTS:
 621                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 622                 ucode_size = BTC_MC_UCODE_SIZE;
 623                 regs_size = BTC_IO_MC_REGS_SIZE;
 624                 break;
 625         case CHIP_TURKS:
 626                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_CAICOS:
 631         default:
 632                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 633                 ucode_size = BTC_MC_UCODE_SIZE;
 634                 regs_size = BTC_IO_MC_REGS_SIZE;
 635                 break;
 636         case CHIP_CAYMAN:
 637                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 638                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         }
 642
 643         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 644         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 645
 646         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 647                 if (running) {
 648                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 649                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 650                 }
 651
 652                 /* reset the engine and set to writable */
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 654                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 655
 656                 /* load mc io regs */
 657                 for (i = 0; i < regs_size; i++) {
 658                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 659                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 660                 }
 661                 /* load the MC ucode */
 662                 fw_data = (const __be32 *)rdev->mc_fw->data;
 663                 for (i = 0; i < ucode_size; i++)
 664                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 665
 666                 /* put the engine back into the active state */
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 669                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 670
 671                 /* wait for training to complete */
 672                 for (i = 0; i < rdev->usec_timeout; i++) {
 673                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 674                                 break;
 675                         udelay(1);
 676                 }
 677
 678                 if (running)
 679                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 680         }
 681
 682         return 0;
 683 }
 684
 685 int ni_init_microcode(struct radeon_device *rdev)
 686 {
 687         const char *chip_name;
 688         const char *rlc_chip_name;
 689         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 690         size_t smc_req_size = 0;
 691         char fw_name[30];
 692         int err;
 693
 694         DRM_DEBUG("\n");
 695
 696         switch (rdev->family) {
 697         case CHIP_BARTS:
 698                 chip_name = "BARTS";
 699                 rlc_chip_name = "BTC";
 700                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 701                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 702                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 703                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 704                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 705                 break;
 706         case CHIP_TURKS:
 707                 chip_name = "TURKS";
 708                 rlc_chip_name = "BTC";
 709                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 710                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 711                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 712                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 713                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 714                 break;
 715         case CHIP_CAICOS:
 716                 chip_name = "CAICOS";
 717                 rlc_chip_name = "BTC";
 718                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 719                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 720                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 721                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 722                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 723                 break;
 724         case CHIP_CAYMAN:
 725                 chip_name = "CAYMAN";
 726                 rlc_chip_name = "CAYMAN";
 727                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 728                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 729                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 730                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 731                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 732                 break;
 733         case CHIP_ARUBA:
 734                 chip_name = "ARUBA";
 735                 rlc_chip_name = "ARUBA";
 736                 /* pfp/me same size as CAYMAN */
 737                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 738                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 739                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 740                 mc_req_size = 0;
 741                 break;
 742         default: BUG();
 743         }
 744
 745         DRM_INFO("Loading %s Microcode\n", chip_name);
 746
 747         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 748         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 749         if (err)
 750                 goto out;
 751         if (rdev->pfp_fw->size != pfp_req_size) {
 752                 printk(KERN_ERR
 753                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 754                        rdev->pfp_fw->size, fw_name);
 755                 err = -EINVAL;
 756                 goto out;
 757         }
 758
 759         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 760         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 761         if (err)
 762                 goto out;
 763         if (rdev->me_fw->size != me_req_size) {
 764                 printk(KERN_ERR
 765                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 766                        rdev->me_fw->size, fw_name);
 767                 err = -EINVAL;
 768         }
 769
 770         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 771         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 772         if (err)
 773                 goto out;
 774         if (rdev->rlc_fw->size != rlc_req_size) {
 775                 printk(KERN_ERR
 776                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 777                        rdev->rlc_fw->size, fw_name);
 778                 err = -EINVAL;
 779         }
 780
 781         /* no MC ucode on TN */
 782         if (!(rdev->flags & RADEON_IS_IGP)) {
 783                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 784                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 785                 if (err)
 786                         goto out;
 787                 if (rdev->mc_fw->size != mc_req_size) {
 788                         printk(KERN_ERR
 789                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 790                                rdev->mc_fw->size, fw_name);
 791                         err = -EINVAL;
 792                 }
 793         }
 794
 795         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 796                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 797                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 798                 if (err) {
 799                         printk(KERN_ERR
 800                                "smc: error loading firmware \"%s\"\n",
 801                                fw_name);
 802                         release_firmware(rdev->smc_fw);
 803                         rdev->smc_fw = NULL;
 804                         err = 0;
 805                 } else if (rdev->smc_fw->size != smc_req_size) {
 806                         printk(KERN_ERR
 807                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 808                                rdev->mc_fw->size, fw_name);
 809                         err = -EINVAL;
 810                 }
 811         }
 812
 813 out:
 814         if (err) {
 815                 if (err != -EINVAL)
 816                         printk(KERN_ERR
 817                                "ni_cp: Failed to load firmware \"%s\"\n",
 818                                fw_name);
 819                 release_firmware(rdev->pfp_fw);
 820                 rdev->pfp_fw = NULL;
 821                 release_firmware(rdev->me_fw);
 822                 rdev->me_fw = NULL;
 823                 release_firmware(rdev->rlc_fw);
 824                 rdev->rlc_fw = NULL;
 825                 release_firmware(rdev->mc_fw);
 826                 rdev->mc_fw = NULL;
 827         }
 828         return err;
 829 }
 830
 831 int tn_get_temp(struct radeon_device *rdev)
 832 {
 833         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 834         int actual_temp = (temp / 8) - 49;
 835
 836         return actual_temp * 1000;
 837 }
 838
 839 /*
 840  * Core functions
 841  */
 842 static void cayman_gpu_init(struct radeon_device *rdev)
 843 {
 844         u32 gb_addr_config = 0;
 845         u32 mc_shared_chmap, mc_arb_ramcfg;
 846         u32 cgts_tcc_disable;
 847         u32 sx_debug_1;
 848         u32 smx_dc_ctl0;
 849         u32 cgts_sm_ctrl_reg;
 850         u32 hdp_host_path_cntl;
 851         u32 tmp;
 852         u32 disabled_rb_mask;
 853         int i, j;
 854
 855         switch (rdev->family) {
 856         case CHIP_CAYMAN:
 857                 rdev->config.cayman.max_shader_engines = 2;
 858                 rdev->config.cayman.max_pipes_per_simd = 4;
 859                 rdev->config.cayman.max_tile_pipes = 8;
 860                 rdev->config.cayman.max_simds_per_se = 12;
 861                 rdev->config.cayman.max_backends_per_se = 4;
 862                 rdev->config.cayman.max_texture_channel_caches = 8;
 863                 rdev->config.cayman.max_gprs = 256;
 864                 rdev->config.cayman.max_threads = 256;
 865                 rdev->config.cayman.max_gs_threads = 32;
 866                 rdev->config.cayman.max_stack_entries = 512;
 867                 rdev->config.cayman.sx_num_of_sets = 8;
 868                 rdev->config.cayman.sx_max_export_size = 256;
 869                 rdev->config.cayman.sx_max_export_pos_size = 64;
 870                 rdev->config.cayman.sx_max_export_smx_size = 192;
 871                 rdev->config.cayman.max_hw_contexts = 8;
 872                 rdev->config.cayman.sq_num_cf_insts = 2;
 873
 874                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 875                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 876                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 877                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 878                 break;
 879         case CHIP_ARUBA:
 880         default:
 881                 rdev->config.cayman.max_shader_engines = 1;
 882                 rdev->config.cayman.max_pipes_per_simd = 4;
 883                 rdev->config.cayman.max_tile_pipes = 2;
 884                 if ((rdev->pdev->device == 0x9900) ||
 885                     (rdev->pdev->device == 0x9901) ||
 886                     (rdev->pdev->device == 0x9905) ||
 887                     (rdev->pdev->device == 0x9906) ||
 888                     (rdev->pdev->device == 0x9907) ||
 889                     (rdev->pdev->device == 0x9908) ||
 890                     (rdev->pdev->device == 0x9909) ||
 891                     (rdev->pdev->device == 0x990B) ||
 892                     (rdev->pdev->device == 0x990C) ||
 893                     (rdev->pdev->device == 0x990F) ||
 894                     (rdev->pdev->device == 0x9910) ||
 895                     (rdev->pdev->device == 0x9917) ||
 896                     (rdev->pdev->device == 0x9999) ||
 897                     (rdev->pdev->device == 0x999C)) {
 898                         rdev->config.cayman.max_simds_per_se = 6;
 899                         rdev->config.cayman.max_backends_per_se = 2;
 900                         rdev->config.cayman.max_hw_contexts = 8;
 901                         rdev->config.cayman.sx_max_export_size = 256;
 902                         rdev->config.cayman.sx_max_export_pos_size = 64;
 903                         rdev->config.cayman.sx_max_export_smx_size = 192;
 904                 } else if ((rdev->pdev->device == 0x9903) ||
 905                            (rdev->pdev->device == 0x9904) ||
 906                            (rdev->pdev->device == 0x990A) ||
 907                            (rdev->pdev->device == 0x990D) ||
 908                            (rdev->pdev->device == 0x990E) ||
 909                            (rdev->pdev->device == 0x9913) ||
 910                            (rdev->pdev->device == 0x9918) ||
 911                            (rdev->pdev->device == 0x999D)) {
 912                         rdev->config.cayman.max_simds_per_se = 4;
 913                         rdev->config.cayman.max_backends_per_se = 2;
 914                         rdev->config.cayman.max_hw_contexts = 8;
 915                         rdev->config.cayman.sx_max_export_size = 256;
 916                         rdev->config.cayman.sx_max_export_pos_size = 64;
 917                         rdev->config.cayman.sx_max_export_smx_size = 192;
 918                 } else if ((rdev->pdev->device == 0x9919) ||
 919                            (rdev->pdev->device == 0x9990) ||
 920                            (rdev->pdev->device == 0x9991) ||
 921                            (rdev->pdev->device == 0x9994) ||
 922                            (rdev->pdev->device == 0x9995) ||
 923                            (rdev->pdev->device == 0x9996) ||
 924                            (rdev->pdev->device == 0x999A) ||
 925                            (rdev->pdev->device == 0x99A0)) {
 926                         rdev->config.cayman.max_simds_per_se = 3;
 927                         rdev->config.cayman.max_backends_per_se = 1;
 928                         rdev->config.cayman.max_hw_contexts = 4;
 929                         rdev->config.cayman.sx_max_export_size = 128;
 930                         rdev->config.cayman.sx_max_export_pos_size = 32;
 931                         rdev->config.cayman.sx_max_export_smx_size = 96;
 932                 } else {
 933                         rdev->config.cayman.max_simds_per_se = 2;
 934                         rdev->config.cayman.max_backends_per_se = 1;
 935                         rdev->config.cayman.max_hw_contexts = 4;
 936                         rdev->config.cayman.sx_max_export_size = 128;
 937                         rdev->config.cayman.sx_max_export_pos_size = 32;
 938                         rdev->config.cayman.sx_max_export_smx_size = 96;
 939                 }
 940                 rdev->config.cayman.max_texture_channel_caches = 2;
 941                 rdev->config.cayman.max_gprs = 256;
 942                 rdev->config.cayman.max_threads = 256;
 943                 rdev->config.cayman.max_gs_threads = 32;
 944                 rdev->config.cayman.max_stack_entries = 512;
 945                 rdev->config.cayman.sx_num_of_sets = 8;
 946                 rdev->config.cayman.sq_num_cf_insts = 2;
 947
 948                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 949                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 950                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 951                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 952                 break;
 953         }
 954
 955         /* Initialize HDP */
 956         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 957                 WREG32((0x2c14 + j), 0x00000000);
 958                 WREG32((0x2c18 + j), 0x00000000);
 959                 WREG32((0x2c1c + j), 0x00000000);
 960                 WREG32((0x2c20 + j), 0x00000000);
 961                 WREG32((0x2c24 + j), 0x00000000);
 962         }
 963
 964         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 965         WREG32(SRBM_INT_CNTL, 0x1);
 966         WREG32(SRBM_INT_ACK, 0x1);
 967
 968         evergreen_fix_pci_max_read_req_size(rdev);
 969
 970         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 971         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 972
 973         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 974         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 975         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 976                 rdev->config.cayman.mem_row_size_in_kb = 4;
 977         /* XXX use MC settings? */
 978         rdev->config.cayman.shader_engine_tile_size = 32;
 979         rdev->config.cayman.num_gpus = 1;
 980         rdev->config.cayman.multi_gpu_tile_size = 64;
 981
 982         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 983         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 984         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 985         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 986         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 987         rdev->config.cayman.num_shader_engines = tmp + 1;
 988         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 989         rdev->config.cayman.num_gpus = tmp + 1;
 990         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 991         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 992         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 993         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 994
 995
 996         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 997          * not have bank info, so create a custom tiling dword.
 998          * bits 3:0   num_pipes
 999          * bits 7:4   num_banks
1000          * bits 11:8  group_size
1001          * bits 15:12 row_size
1002          */
1003         rdev->config.cayman.tile_config = 0;
1004         switch (rdev->config.cayman.num_tile_pipes) {
1005         case 1:
1006         default:
1007                 rdev->config.cayman.tile_config |= (0 << 0);
1008                 break;
1009         case 2:
1010                 rdev->config.cayman.tile_config |= (1 << 0);
1011                 break;
1012         case 4:
1013                 rdev->config.cayman.tile_config |= (2 << 0);
1014                 break;
1015         case 8:
1016                 rdev->config.cayman.tile_config |= (3 << 0);
1017                 break;
1018         }
1019
1020         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1021         if (rdev->flags & RADEON_IS_IGP)
1022                 rdev->config.cayman.tile_config |= 1 << 4;
1023         else {
1024                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1025                 case 0: /* four banks */
1026                         rdev->config.cayman.tile_config |= 0 << 4;
1027                         break;
1028                 case 1: /* eight banks */
1029                         rdev->config.cayman.tile_config |= 1 << 4;
1030                         break;
1031                 case 2: /* sixteen banks */
1032                 default:
1033                         rdev->config.cayman.tile_config |= 2 << 4;
1034                         break;
1035                 }
1036         }
1037         rdev->config.cayman.tile_config |=
1038                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1039         rdev->config.cayman.tile_config |=
1040                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1041
1042         tmp = 0;
1043         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1044                 u32 rb_disable_bitmap;
1045
1046                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1047                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1048                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1049                 tmp <<= 4;
1050                 tmp |= rb_disable_bitmap;
1051         }
1052         /* enabled rb are just the one not disabled :) */
1053         disabled_rb_mask = tmp;
1054         tmp = 0;
1055         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1056                 tmp |= (1 << i);
1057         /* if all the backends are disabled, fix it up here */
1058         if ((disabled_rb_mask & tmp) == tmp) {
1059                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1060                         disabled_rb_mask &= ~(1 << i);
1061         }
1062
1063         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1064                 u32 simd_disable_bitmap;
1065
1066                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1067                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1068                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1069                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1070                 tmp <<= 16;
1071                 tmp |= simd_disable_bitmap;
1072         }
1073         rdev->config.cayman.active_simds = hweight32(~tmp);
1074
1075         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1076         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1077
1078         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1079         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1080         if (ASIC_IS_DCE6(rdev))
1081                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1082         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1083         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1084         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1085         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1086         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1087         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1088
1089         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1090             (rdev->flags & RADEON_IS_IGP)) {
1091                 if ((disabled_rb_mask & 3) == 2) {
1092                         /* RB1 disabled, RB0 enabled */
1093                         tmp = 0x00000000;
1094                 } else {
1095                         /* RB0 disabled, RB1 enabled */
1096                         tmp = 0x11111111;
1097                 }
1098         } else {
1099                 tmp = gb_addr_config & NUM_PIPES_MASK;
1100                 tmp = r6xx_remap_render_backend(rdev, tmp,
1101                                                 rdev->config.cayman.max_backends_per_se *
1102                                                 rdev->config.cayman.max_shader_engines,
1103                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1104         }
1105         WREG32(GB_BACKEND_MAP, tmp);
1106
1107         cgts_tcc_disable = 0xffff0000;
1108         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1109                 cgts_tcc_disable &= ~(1 << (16 + i));
1110         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1111         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1112         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1113         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1114
1115         /* reprogram the shader complex */
1116         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1117         for (i = 0; i < 16; i++)
1118                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1119         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1120
1121         /* set HW defaults for 3D engine */
1122         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1123
1124         sx_debug_1 = RREG32(SX_DEBUG_1);
1125         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1126         WREG32(SX_DEBUG_1, sx_debug_1);
1127
1128         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1129         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1130         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1131         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1132
1133         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1134
1135         /* need to be explicitly zero-ed */
1136         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1137         WREG32(SQ_LSTMP_RING_BASE, 0);
1138         WREG32(SQ_HSTMP_RING_BASE, 0);
1139         WREG32(SQ_ESTMP_RING_BASE, 0);
1140         WREG32(SQ_GSTMP_RING_BASE, 0);
1141         WREG32(SQ_VSTMP_RING_BASE, 0);
1142         WREG32(SQ_PSTMP_RING_BASE, 0);
1143
1144         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1145
1146         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1147                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1148                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1149
1150         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1151                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1152                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1153
1154
1155         WREG32(VGT_NUM_INSTANCES, 1);
1156
1157         WREG32(CP_PERFMON_CNTL, 0);
1158
1159         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1160                                   FETCH_FIFO_HIWATER(0x4) |
1161                                   DONE_FIFO_HIWATER(0xe0) |
1162                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1163
1164         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1165         WREG32(SQ_CONFIG, (VC_ENABLE |
1166                            EXPORT_SRC_C |
1167                            GFX_PRIO(0) |
1168                            CS1_PRIO(0) |
1169                            CS2_PRIO(1)));
1170         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1171
1172         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1173                                           FORCE_EOV_MAX_REZ_CNT(255)));
1174
1175         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1176                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1177
1178         WREG32(VGT_GS_VERTEX_REUSE, 16);
1179         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1180
1181         WREG32(CB_PERF_CTR0_SEL_0, 0);
1182         WREG32(CB_PERF_CTR0_SEL_1, 0);
1183         WREG32(CB_PERF_CTR1_SEL_0, 0);
1184         WREG32(CB_PERF_CTR1_SEL_1, 0);
1185         WREG32(CB_PERF_CTR2_SEL_0, 0);
1186         WREG32(CB_PERF_CTR2_SEL_1, 0);
1187         WREG32(CB_PERF_CTR3_SEL_0, 0);
1188         WREG32(CB_PERF_CTR3_SEL_1, 0);
1189
1190         tmp = RREG32(HDP_MISC_CNTL);
1191         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1192         WREG32(HDP_MISC_CNTL, tmp);
1193
1194         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1195         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1196
1197         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1198
1199         udelay(50);
1200
1201         /* set clockgating golden values on TN */
1202         if (rdev->family == CHIP_ARUBA) {
1203                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1204                 tmp &= ~0x00380000;
1205                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1206                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1207                 tmp &= ~0x0e000000;
1208                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1209         }
1210 }
1211
1212 /*
1213  * GART
1214  */
1215 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1216 {
1217         /* flush hdp cache */
1218         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1219
1220         /* bits 0-7 are the VM contexts0-7 */
1221         WREG32(VM_INVALIDATE_REQUEST, 1);
1222 }
1223
1224 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1225 {
1226         int i, r;
1227
1228         if (rdev->gart.robj == NULL) {
1229                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1230                 return -EINVAL;
1231         }
1232         r = radeon_gart_table_vram_pin(rdev);
1233         if (r)
1234                 return r;
1235         /* Setup TLB control */
1236         WREG32(MC_VM_MX_L1_TLB_CNTL,
1237                (0xA << 7) |
1238                ENABLE_L1_TLB |
1239                ENABLE_L1_FRAGMENT_PROCESSING |
1240                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1241                ENABLE_ADVANCED_DRIVER_MODEL |
1242                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1243         /* Setup L2 cache */
1244         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1245                ENABLE_L2_FRAGMENT_PROCESSING |
1246                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1247                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1248                EFFECTIVE_L2_QUEUE_SIZE(7) |
1249                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1250         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1251         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1252                BANK_SELECT(6) |
1253                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1254         /* setup context0 */
1255         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1256         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1257         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1258         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1259                         (u32)(rdev->dummy_page.addr >> 12));
1260         WREG32(VM_CONTEXT0_CNTL2, 0);
1261         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1262                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1263
1264         WREG32(0x15D4, 0);
1265         WREG32(0x15D8, 0);
1266         WREG32(0x15DC, 0);
1267
1268         /* empty context1-7 */
1269         /* Assign the pt base to something valid for now; the pts used for
1270          * the VMs are determined by the application and setup and assigned
1271          * on the fly in the vm part of radeon_gart.c
1272          */
1273         for (i = 1; i < 8; i++) {
1274                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1275                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1276                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1277                        rdev->vm_manager.saved_table_addr[i]);
1278         }
1279
1280         /* enable context1-7 */
1281         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1282                (u32)(rdev->dummy_page.addr >> 12));
1283         WREG32(VM_CONTEXT1_CNTL2, 4);
1284         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1285                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1286                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1287                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1288                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1289                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1290                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1291                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1292                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1293                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1294                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1295                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1296                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1297                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1298
1299         cayman_pcie_gart_tlb_flush(rdev);
1300         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1301                  (unsigned)(rdev->mc.gtt_size >> 20),
1302                  (unsigned long long)rdev->gart.table_addr);
1303         rdev->gart.ready = true;
1304         return 0;
1305 }
1306
1307 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1308 {
1309         unsigned i;
1310
1311         for (i = 1; i < 8; ++i) {
1312                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1313                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1314         }
1315
1316         /* Disable all tables */
1317         WREG32(VM_CONTEXT0_CNTL, 0);
1318         WREG32(VM_CONTEXT1_CNTL, 0);
1319         /* Setup TLB control */
1320         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1321                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1322                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1323         /* Setup L2 cache */
1324         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1325                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1326                EFFECTIVE_L2_QUEUE_SIZE(7) |
1327                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1328         WREG32(VM_L2_CNTL2, 0);
1329         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1330                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1331         radeon_gart_table_vram_unpin(rdev);
1332 }
1333
1334 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1335 {
1336         cayman_pcie_gart_disable(rdev);
1337         radeon_gart_table_vram_free(rdev);
1338         radeon_gart_fini(rdev);
1339 }
1340
1341 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1342                               int ring, u32 cp_int_cntl)
1343 {
1344         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1345
1346         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1347         WREG32(CP_INT_CNTL, cp_int_cntl);
1348 }
1349
1350 /*
1351  * CP.
1352  */
1353 void cayman_fence_ring_emit(struct radeon_device *rdev,
1354                             struct radeon_fence *fence)
1355 {
1356         struct radeon_ring *ring = &rdev->ring[fence->ring];
1357         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1358         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1359                 PACKET3_SH_ACTION_ENA;
1360
1361         /* flush read cache over gart for this vmid */
1362         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1363         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1364         radeon_ring_write(ring, 0xFFFFFFFF);
1365         radeon_ring_write(ring, 0);
1366         radeon_ring_write(ring, 10); /* poll interval */
1367         /* EVENT_WRITE_EOP - flush caches, send int */
1368         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1369         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1370         radeon_ring_write(ring, lower_32_bits(addr));
1371         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1372         radeon_ring_write(ring, fence->seq);
1373         radeon_ring_write(ring, 0);
1374 }
1375
1376 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1377 {
1378         struct radeon_ring *ring = &rdev->ring[ib->ring];
1379         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1380         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1381                 PACKET3_SH_ACTION_ENA;
1382
1383         /* set to DX10/11 mode */
1384         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1385         radeon_ring_write(ring, 1);
1386
1387         if (ring->rptr_save_reg) {
1388                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1389                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1390                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1391                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1392                 radeon_ring_write(ring, next_rptr);
1393         }
1394
1395         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1396         radeon_ring_write(ring,
1397 #ifdef __BIG_ENDIAN
1398                           (2 << 0) |
1399 #endif
1400                           (ib->gpu_addr & 0xFFFFFFFC));
1401         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1402         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1403
1404         /* flush read cache over gart for this vmid */
1405         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1406         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1407         radeon_ring_write(ring, 0xFFFFFFFF);
1408         radeon_ring_write(ring, 0);
1409         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1410 }
1411
1412 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1413 {
1414         if (enable)
1415                 WREG32(CP_ME_CNTL, 0);
1416         else {
1417                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1418                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1419                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1420                 WREG32(SCRATCH_UMSK, 0);
1421                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1422         }
1423 }
1424
1425 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1426                         struct radeon_ring *ring)
1427 {
1428         u32 rptr;
1429
1430         if (rdev->wb.enabled)
1431                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1432         else {
1433                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1434                         rptr = RREG32(CP_RB0_RPTR);
1435                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1436                         rptr = RREG32(CP_RB1_RPTR);
1437                 else
1438                         rptr = RREG32(CP_RB2_RPTR);
1439         }
1440
1441         return rptr;
1442 }
1443
1444 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1445                         struct radeon_ring *ring)
1446 {
1447         u32 wptr;
1448
1449         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1450                 wptr = RREG32(CP_RB0_WPTR);
1451         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1452                 wptr = RREG32(CP_RB1_WPTR);
1453         else
1454                 wptr = RREG32(CP_RB2_WPTR);
1455
1456         return wptr;
1457 }
1458
1459 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1460                          struct radeon_ring *ring)
1461 {
1462         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1463                 WREG32(CP_RB0_WPTR, ring->wptr);
1464                 (void)RREG32(CP_RB0_WPTR);
1465         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1466                 WREG32(CP_RB1_WPTR, ring->wptr);
1467                 (void)RREG32(CP_RB1_WPTR);
1468         } else {
1469                 WREG32(CP_RB2_WPTR, ring->wptr);
1470                 (void)RREG32(CP_RB2_WPTR);
1471         }
1472 }
1473
1474 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1475 {
1476         const __be32 *fw_data;
1477         int i;
1478
1479         if (!rdev->me_fw || !rdev->pfp_fw)
1480                 return -EINVAL;
1481
1482         cayman_cp_enable(rdev, false);
1483
1484         fw_data = (const __be32 *)rdev->pfp_fw->data;
1485         WREG32(CP_PFP_UCODE_ADDR, 0);
1486         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1487                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1488         WREG32(CP_PFP_UCODE_ADDR, 0);
1489
1490         fw_data = (const __be32 *)rdev->me_fw->data;
1491         WREG32(CP_ME_RAM_WADDR, 0);
1492         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1493                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1494
1495         WREG32(CP_PFP_UCODE_ADDR, 0);
1496         WREG32(CP_ME_RAM_WADDR, 0);
1497         WREG32(CP_ME_RAM_RADDR, 0);
1498         return 0;
1499 }
1500
1501 static int cayman_cp_start(struct radeon_device *rdev)
1502 {
1503         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1504         int r, i;
1505
1506         r = radeon_ring_lock(rdev, ring, 7);
1507         if (r) {
1508                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1509                 return r;
1510         }
1511         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1512         radeon_ring_write(ring, 0x1);
1513         radeon_ring_write(ring, 0x0);
1514         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1515         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1516         radeon_ring_write(ring, 0);
1517         radeon_ring_write(ring, 0);
1518         radeon_ring_unlock_commit(rdev, ring, false);
1519
1520         cayman_cp_enable(rdev, true);
1521
1522         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1523         if (r) {
1524                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1525                 return r;
1526         }
1527
1528         /* setup clear context state */
1529         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1530         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1531
1532         for (i = 0; i < cayman_default_size; i++)
1533                 radeon_ring_write(ring, cayman_default_state[i]);
1534
1535         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1536         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1537
1538         /* set clear context state */
1539         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1540         radeon_ring_write(ring, 0);
1541
1542         /* SQ_VTX_BASE_VTX_LOC */
1543         radeon_ring_write(ring, 0xc0026f00);
1544         radeon_ring_write(ring, 0x00000000);
1545         radeon_ring_write(ring, 0x00000000);
1546         radeon_ring_write(ring, 0x00000000);
1547
1548         /* Clear consts */
1549         radeon_ring_write(ring, 0xc0036f00);
1550         radeon_ring_write(ring, 0x00000bc4);
1551         radeon_ring_write(ring, 0xffffffff);
1552         radeon_ring_write(ring, 0xffffffff);
1553         radeon_ring_write(ring, 0xffffffff);
1554
1555         radeon_ring_write(ring, 0xc0026900);
1556         radeon_ring_write(ring, 0x00000316);
1557         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1558         radeon_ring_write(ring, 0x00000010); /*  */
1559
1560         radeon_ring_unlock_commit(rdev, ring, false);
1561
1562         /* XXX init other rings */
1563
1564         return 0;
1565 }
1566
1567 static void cayman_cp_fini(struct radeon_device *rdev)
1568 {
1569         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1570         cayman_cp_enable(rdev, false);
1571         radeon_ring_fini(rdev, ring);
1572         radeon_scratch_free(rdev, ring->rptr_save_reg);
1573 }
1574
1575 static int cayman_cp_resume(struct radeon_device *rdev)
1576 {
1577         static const int ridx[] = {
1578                 RADEON_RING_TYPE_GFX_INDEX,
1579                 CAYMAN_RING_TYPE_CP1_INDEX,
1580                 CAYMAN_RING_TYPE_CP2_INDEX
1581         };
1582         static const unsigned cp_rb_cntl[] = {
1583                 CP_RB0_CNTL,
1584                 CP_RB1_CNTL,
1585                 CP_RB2_CNTL,
1586         };
1587         static const unsigned cp_rb_rptr_addr[] = {
1588                 CP_RB0_RPTR_ADDR,
1589                 CP_RB1_RPTR_ADDR,
1590                 CP_RB2_RPTR_ADDR
1591         };
1592         static const unsigned cp_rb_rptr_addr_hi[] = {
1593                 CP_RB0_RPTR_ADDR_HI,
1594                 CP_RB1_RPTR_ADDR_HI,
1595                 CP_RB2_RPTR_ADDR_HI
1596         };
1597         static const unsigned cp_rb_base[] = {
1598                 CP_RB0_BASE,
1599                 CP_RB1_BASE,
1600                 CP_RB2_BASE
1601         };
1602         static const unsigned cp_rb_rptr[] = {
1603                 CP_RB0_RPTR,
1604                 CP_RB1_RPTR,
1605                 CP_RB2_RPTR
1606         };
1607         static const unsigned cp_rb_wptr[] = {
1608                 CP_RB0_WPTR,
1609                 CP_RB1_WPTR,
1610                 CP_RB2_WPTR
1611         };
1612         struct radeon_ring *ring;
1613         int i, r;
1614
1615         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1616         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1617                                  SOFT_RESET_PA |
1618                                  SOFT_RESET_SH |
1619                                  SOFT_RESET_VGT |
1620                                  SOFT_RESET_SPI |
1621                                  SOFT_RESET_SX));
1622         RREG32(GRBM_SOFT_RESET);
1623         mdelay(15);
1624         WREG32(GRBM_SOFT_RESET, 0);
1625         RREG32(GRBM_SOFT_RESET);
1626
1627         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1628         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1629
1630         /* Set the write pointer delay */
1631         WREG32(CP_RB_WPTR_DELAY, 0);
1632
1633         WREG32(CP_DEBUG, (1 << 27));
1634
1635         /* set the wb address whether it's enabled or not */
1636         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1637         WREG32(SCRATCH_UMSK, 0xff);
1638
1639         for (i = 0; i < 3; ++i) {
1640                 uint32_t rb_cntl;
1641                 uint64_t addr;
1642
1643                 /* Set ring buffer size */
1644                 ring = &rdev->ring[ridx[i]];
1645                 rb_cntl = order_base_2(ring->ring_size / 8);
1646                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1647 #ifdef __BIG_ENDIAN
1648                 rb_cntl |= BUF_SWAP_32BIT;
1649 #endif
1650                 WREG32(cp_rb_cntl[i], rb_cntl);
1651
1652                 /* set the wb address whether it's enabled or not */
1653                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1654                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1655                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1656         }
1657
1658         /* set the rb base addr, this causes an internal reset of ALL rings */
1659         for (i = 0; i < 3; ++i) {
1660                 ring = &rdev->ring[ridx[i]];
1661                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1662         }
1663
1664         for (i = 0; i < 3; ++i) {
1665                 /* Initialize the ring buffer's read and write pointers */
1666                 ring = &rdev->ring[ridx[i]];
1667                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1668
1669                 ring->wptr = 0;
1670                 WREG32(cp_rb_rptr[i], 0);
1671                 WREG32(cp_rb_wptr[i], ring->wptr);
1672
1673                 mdelay(1);
1674                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1675         }
1676
1677         /* start the rings */
1678         cayman_cp_start(rdev);
1679         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1680         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1681         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1682         /* this only test cp0 */
1683         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1684         if (r) {
1685                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1686                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1687                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1688                 return r;
1689         }
1690
1691         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1692                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1693
1694         return 0;
1695 }
1696
1697 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1698 {
1699         u32 reset_mask = 0;
1700         u32 tmp;
1701
1702         /* GRBM_STATUS */
1703         tmp = RREG32(GRBM_STATUS);
1704         if (tmp & (PA_BUSY | SC_BUSY |
1705                    SH_BUSY | SX_BUSY |
1706                    TA_BUSY | VGT_BUSY |
1707                    DB_BUSY | CB_BUSY |
1708                    GDS_BUSY | SPI_BUSY |
1709                    IA_BUSY | IA_BUSY_NO_DMA))
1710                 reset_mask |= RADEON_RESET_GFX;
1711
1712         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1713                    CP_BUSY | CP_COHERENCY_BUSY))
1714                 reset_mask |= RADEON_RESET_CP;
1715
1716         if (tmp & GRBM_EE_BUSY)
1717                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1718
1719         /* DMA_STATUS_REG 0 */
1720         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1721         if (!(tmp & DMA_IDLE))
1722                 reset_mask |= RADEON_RESET_DMA;
1723
1724         /* DMA_STATUS_REG 1 */
1725         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1726         if (!(tmp & DMA_IDLE))
1727                 reset_mask |= RADEON_RESET_DMA1;
1728
1729         /* SRBM_STATUS2 */
1730         tmp = RREG32(SRBM_STATUS2);
1731         if (tmp & DMA_BUSY)
1732                 reset_mask |= RADEON_RESET_DMA;
1733
1734         if (tmp & DMA1_BUSY)
1735                 reset_mask |= RADEON_RESET_DMA1;
1736
1737         /* SRBM_STATUS */
1738         tmp = RREG32(SRBM_STATUS);
1739         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1740                 reset_mask |= RADEON_RESET_RLC;
1741
1742         if (tmp & IH_BUSY)
1743                 reset_mask |= RADEON_RESET_IH;
1744
1745         if (tmp & SEM_BUSY)
1746                 reset_mask |= RADEON_RESET_SEM;
1747
1748         if (tmp & GRBM_RQ_PENDING)
1749                 reset_mask |= RADEON_RESET_GRBM;
1750
1751         if (tmp & VMC_BUSY)
1752                 reset_mask |= RADEON_RESET_VMC;
1753
1754         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1755                    MCC_BUSY | MCD_BUSY))
1756                 reset_mask |= RADEON_RESET_MC;
1757
1758         if (evergreen_is_display_hung(rdev))
1759                 reset_mask |= RADEON_RESET_DISPLAY;
1760
1761         /* VM_L2_STATUS */
1762         tmp = RREG32(VM_L2_STATUS);
1763         if (tmp & L2_BUSY)
1764                 reset_mask |= RADEON_RESET_VMC;
1765
1766         /* Skip MC reset as it's mostly likely not hung, just busy */
1767         if (reset_mask & RADEON_RESET_MC) {
1768                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1769                 reset_mask &= ~RADEON_RESET_MC;
1770         }
1771
1772         return reset_mask;
1773 }
1774
1775 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1776 {
1777         struct evergreen_mc_save save;
1778         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1779         u32 tmp;
1780
1781         if (reset_mask == 0)
1782                 return;
1783
1784         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1785
1786         evergreen_print_gpu_status_regs(rdev);
1787         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1788                  RREG32(0x14F8));
1789         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1790                  RREG32(0x14D8));
1791         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1792                  RREG32(0x14FC));
1793         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1794                  RREG32(0x14DC));
1795
1796         /* Disable CP parsing/prefetching */
1797         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1798
1799         if (reset_mask & RADEON_RESET_DMA) {
1800                 /* dma0 */
1801                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1802                 tmp &= ~DMA_RB_ENABLE;
1803                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1804         }
1805
1806         if (reset_mask & RADEON_RESET_DMA1) {
1807                 /* dma1 */
1808                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1809                 tmp &= ~DMA_RB_ENABLE;
1810                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1811         }
1812
1813         udelay(50);
1814
1815         evergreen_mc_stop(rdev, &save);
1816         if (evergreen_mc_wait_for_idle(rdev)) {
1817                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1818         }
1819
1820         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1821                 grbm_soft_reset = SOFT_RESET_CB |
1822                         SOFT_RESET_DB |
1823                         SOFT_RESET_GDS |
1824                         SOFT_RESET_PA |
1825                         SOFT_RESET_SC |
1826                         SOFT_RESET_SPI |
1827                         SOFT_RESET_SH |
1828                         SOFT_RESET_SX |
1829                         SOFT_RESET_TC |
1830                         SOFT_RESET_TA |
1831                         SOFT_RESET_VGT |
1832                         SOFT_RESET_IA;
1833         }
1834
1835         if (reset_mask & RADEON_RESET_CP) {
1836                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1837
1838                 srbm_soft_reset |= SOFT_RESET_GRBM;
1839         }
1840
1841         if (reset_mask & RADEON_RESET_DMA)
1842                 srbm_soft_reset |= SOFT_RESET_DMA;
1843
1844         if (reset_mask & RADEON_RESET_DMA1)
1845                 srbm_soft_reset |= SOFT_RESET_DMA1;
1846
1847         if (reset_mask & RADEON_RESET_DISPLAY)
1848                 srbm_soft_reset |= SOFT_RESET_DC;
1849
1850         if (reset_mask & RADEON_RESET_RLC)
1851                 srbm_soft_reset |= SOFT_RESET_RLC;
1852
1853         if (reset_mask & RADEON_RESET_SEM)
1854                 srbm_soft_reset |= SOFT_RESET_SEM;
1855
1856         if (reset_mask & RADEON_RESET_IH)
1857                 srbm_soft_reset |= SOFT_RESET_IH;
1858
1859         if (reset_mask & RADEON_RESET_GRBM)
1860                 srbm_soft_reset |= SOFT_RESET_GRBM;
1861
1862         if (reset_mask & RADEON_RESET_VMC)
1863                 srbm_soft_reset |= SOFT_RESET_VMC;
1864
1865         if (!(rdev->flags & RADEON_IS_IGP)) {
1866                 if (reset_mask & RADEON_RESET_MC)
1867                         srbm_soft_reset |= SOFT_RESET_MC;
1868         }
1869
1870         if (grbm_soft_reset) {
1871                 tmp = RREG32(GRBM_SOFT_RESET);
1872                 tmp |= grbm_soft_reset;
1873                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1874                 WREG32(GRBM_SOFT_RESET, tmp);
1875                 tmp = RREG32(GRBM_SOFT_RESET);
1876
1877                 udelay(50);
1878
1879                 tmp &= ~grbm_soft_reset;
1880                 WREG32(GRBM_SOFT_RESET, tmp);
1881                 tmp = RREG32(GRBM_SOFT_RESET);
1882         }
1883
1884         if (srbm_soft_reset) {
1885                 tmp = RREG32(SRBM_SOFT_RESET);
1886                 tmp |= srbm_soft_reset;
1887                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1888                 WREG32(SRBM_SOFT_RESET, tmp);
1889                 tmp = RREG32(SRBM_SOFT_RESET);
1890
1891                 udelay(50);
1892
1893                 tmp &= ~srbm_soft_reset;
1894                 WREG32(SRBM_SOFT_RESET, tmp);
1895                 tmp = RREG32(SRBM_SOFT_RESET);
1896         }
1897
1898         /* Wait a little for things to settle down */
1899         udelay(50);
1900
1901         evergreen_mc_resume(rdev, &save);
1902         udelay(50);
1903
1904         evergreen_print_gpu_status_regs(rdev);
1905 }
1906
1907 int cayman_asic_reset(struct radeon_device *rdev)
1908 {
1909         u32 reset_mask;
1910
1911         reset_mask = cayman_gpu_check_soft_reset(rdev);
1912
1913         if (reset_mask)
1914                 r600_set_bios_scratch_engine_hung(rdev, true);
1915
1916         cayman_gpu_soft_reset(rdev, reset_mask);
1917
1918         reset_mask = cayman_gpu_check_soft_reset(rdev);
1919
1920         if (reset_mask)
1921                 evergreen_gpu_pci_config_reset(rdev);
1922
1923         r600_set_bios_scratch_engine_hung(rdev, false);
1924
1925         return 0;
1926 }
1927
1928 /**
1929  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1930  *
1931  * @rdev: radeon_device pointer
1932  * @ring: radeon_ring structure holding ring information
1933  *
1934  * Check if the GFX engine is locked up.
1935  * Returns true if the engine appears to be locked up, false if not.
1936  */
1937 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1938 {
1939         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1940
1941         if (!(reset_mask & (RADEON_RESET_GFX |
1942                             RADEON_RESET_COMPUTE |
1943                             RADEON_RESET_CP))) {
1944                 radeon_ring_lockup_update(rdev, ring);
1945                 return false;
1946         }
1947         return radeon_ring_test_lockup(rdev, ring);
1948 }
1949
1950 static int cayman_startup(struct radeon_device *rdev)
1951 {
1952         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1953         int r;
1954
1955         /* enable pcie gen2 link */
1956         evergreen_pcie_gen2_enable(rdev);
1957         /* enable aspm */
1958         evergreen_program_aspm(rdev);
1959
1960         /* scratch needs to be initialized before MC */
1961         r = r600_vram_scratch_init(rdev);
1962         if (r)
1963                 return r;
1964
1965         evergreen_mc_program(rdev);
1966
1967         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1968                 r = ni_mc_load_microcode(rdev);
1969                 if (r) {
1970                         DRM_ERROR("Failed to load MC firmware!\n");
1971                         return r;
1972                 }
1973         }
1974
1975         r = cayman_pcie_gart_enable(rdev);
1976         if (r)
1977                 return r;
1978         cayman_gpu_init(rdev);
1979
1980         /* allocate rlc buffers */
1981         if (rdev->flags & RADEON_IS_IGP) {
1982                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1983                 rdev->rlc.reg_list_size =
1984                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1985                 rdev->rlc.cs_data = cayman_cs_data;
1986                 r = sumo_rlc_init(rdev);
1987                 if (r) {
1988                         DRM_ERROR("Failed to init rlc BOs!\n");
1989                         return r;
1990                 }
1991         }
1992
1993         /* allocate wb buffer */
1994         r = radeon_wb_init(rdev);
1995         if (r)
1996                 return r;
1997
1998         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1999         if (r) {
2000                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2001                 return r;
2002         }
2003
2004         r = uvd_v2_2_resume(rdev);
2005         if (!r) {
2006                 r = radeon_fence_driver_start_ring(rdev,
2007                                                    R600_RING_TYPE_UVD_INDEX);
2008                 if (r)
2009                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2010         }
2011         if (r)
2012                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2013
2014         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2015         if (r) {
2016                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2017                 return r;
2018         }
2019
2020         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2021         if (r) {
2022                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2023                 return r;
2024         }
2025
2026         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2027         if (r) {
2028                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2029                 return r;
2030         }
2031
2032         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2033         if (r) {
2034                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2035                 return r;
2036         }
2037
2038         /* Enable IRQ */
2039         if (!rdev->irq.installed) {
2040                 r = radeon_irq_kms_init(rdev);
2041                 if (r)
2042                         return r;
2043         }
2044
2045         r = r600_irq_init(rdev);
2046         if (r) {
2047                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2048                 radeon_irq_kms_fini(rdev);
2049                 return r;
2050         }
2051         evergreen_irq_set(rdev);
2052
2053         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2054                              RADEON_CP_PACKET2);
2055         if (r)
2056                 return r;
2057
2058         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2059         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2060                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2061         if (r)
2062                 return r;
2063
2064         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2065         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2066                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2067         if (r)
2068                 return r;
2069
2070         r = cayman_cp_load_microcode(rdev);
2071         if (r)
2072                 return r;
2073         r = cayman_cp_resume(rdev);
2074         if (r)
2075                 return r;
2076
2077         r = cayman_dma_resume(rdev);
2078         if (r)
2079                 return r;
2080
2081         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2082         if (ring->ring_size) {
2083                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2084                                      RADEON_CP_PACKET2);
2085                 if (!r)
2086                         r = uvd_v1_0_init(rdev);
2087                 if (r)
2088                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2089         }
2090
2091         r = radeon_ib_pool_init(rdev);
2092         if (r) {
2093                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2094                 return r;
2095         }
2096
2097         r = radeon_vm_manager_init(rdev);
2098         if (r) {
2099                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2100                 return r;
2101         }
2102
2103         r = radeon_audio_init(rdev);
2104         if (r)
2105                 return r;
2106
2107         return 0;
2108 }
2109
2110 int cayman_resume(struct radeon_device *rdev)
2111 {
2112         int r;
2113
2114         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2115          * posting will perform necessary task to bring back GPU into good
2116          * shape.
2117          */
2118         /* post card */
2119         atom_asic_init(rdev->mode_info.atom_context);
2120
2121         /* init golden registers */
2122         ni_init_golden_registers(rdev);
2123
2124         if (rdev->pm.pm_method == PM_METHOD_DPM)
2125                 radeon_pm_resume(rdev);
2126
2127         rdev->accel_working = true;
2128         r = cayman_startup(rdev);
2129         if (r) {
2130                 DRM_ERROR("cayman startup failed on resume\n");
2131                 rdev->accel_working = false;
2132                 return r;
2133         }
2134         return r;
2135 }
2136
2137 int cayman_suspend(struct radeon_device *rdev)
2138 {
2139         radeon_pm_suspend(rdev);
2140         radeon_audio_fini(rdev);
2141         radeon_vm_manager_fini(rdev);
2142         cayman_cp_enable(rdev, false);
2143         cayman_dma_stop(rdev);
2144         uvd_v1_0_fini(rdev);
2145         radeon_uvd_suspend(rdev);
2146         evergreen_irq_suspend(rdev);
2147         radeon_wb_disable(rdev);
2148         cayman_pcie_gart_disable(rdev);
2149         return 0;
2150 }
2151
2152 /* Plan is to move initialization in that function and use
2153  * helper function so that radeon_device_init pretty much
2154  * do nothing more than calling asic specific function. This
2155  * should also allow to remove a bunch of callback function
2156  * like vram_info.
2157  */
2158 int cayman_init(struct radeon_device *rdev)
2159 {
2160         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2161         int r;
2162
2163         /* Read BIOS */
2164         if (!radeon_get_bios(rdev)) {
2165                 if (ASIC_IS_AVIVO(rdev))
2166                         return -EINVAL;
2167         }
2168         /* Must be an ATOMBIOS */
2169         if (!rdev->is_atom_bios) {
2170                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2171                 return -EINVAL;
2172         }
2173         r = radeon_atombios_init(rdev);
2174         if (r)
2175                 return r;
2176
2177         /* Post card if necessary */
2178         if (!radeon_card_posted(rdev)) {
2179                 if (!rdev->bios) {
2180                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2181                         return -EINVAL;
2182                 }
2183                 DRM_INFO("GPU not posted. posting now...\n");
2184                 atom_asic_init(rdev->mode_info.atom_context);
2185         }
2186         /* init golden registers */
2187         ni_init_golden_registers(rdev);
2188         /* Initialize scratch registers */
2189         r600_scratch_init(rdev);
2190         /* Initialize surface registers */
2191         radeon_surface_init(rdev);
2192         /* Initialize clocks */
2193         radeon_get_clock_info(rdev->ddev);
2194         /* Fence driver */
2195         r = radeon_fence_driver_init(rdev);
2196         if (r)
2197                 return r;
2198         /* initialize memory controller */
2199         r = evergreen_mc_init(rdev);
2200         if (r)
2201                 return r;
2202         /* Memory manager */
2203         r = radeon_bo_init(rdev);
2204         if (r)
2205                 return r;
2206
2207         if (rdev->flags & RADEON_IS_IGP) {
2208                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2209                         r = ni_init_microcode(rdev);
2210                         if (r) {
2211                                 DRM_ERROR("Failed to load firmware!\n");
2212                                 return r;
2213                         }
2214                 }
2215         } else {
2216                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2217                         r = ni_init_microcode(rdev);
2218                         if (r) {
2219                                 DRM_ERROR("Failed to load firmware!\n");
2220                                 return r;
2221                         }
2222                 }
2223         }
2224
2225         /* Initialize power management */
2226         radeon_pm_init(rdev);
2227
2228         ring->ring_obj = NULL;
2229         r600_ring_init(rdev, ring, 1024 * 1024);
2230
2231         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2232         ring->ring_obj = NULL;
2233         r600_ring_init(rdev, ring, 64 * 1024);
2234
2235         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2236         ring->ring_obj = NULL;
2237         r600_ring_init(rdev, ring, 64 * 1024);
2238
2239         r = radeon_uvd_init(rdev);
2240         if (!r) {
2241                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2242                 ring->ring_obj = NULL;
2243                 r600_ring_init(rdev, ring, 4096);
2244         }
2245
2246         rdev->ih.ring_obj = NULL;
2247         r600_ih_ring_init(rdev, 64 * 1024);
2248
2249         r = r600_pcie_gart_init(rdev);
2250         if (r)
2251                 return r;
2252
2253         rdev->accel_working = true;
2254         r = cayman_startup(rdev);
2255         if (r) {
2256                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2257                 cayman_cp_fini(rdev);
2258                 cayman_dma_fini(rdev);
2259                 r600_irq_fini(rdev);
2260                 if (rdev->flags & RADEON_IS_IGP)
2261                         sumo_rlc_fini(rdev);
2262                 radeon_wb_fini(rdev);
2263                 radeon_ib_pool_fini(rdev);
2264                 radeon_vm_manager_fini(rdev);
2265                 radeon_irq_kms_fini(rdev);
2266                 cayman_pcie_gart_fini(rdev);
2267                 rdev->accel_working = false;
2268         }
2269
2270         /* Don't start up if the MC ucode is missing.
2271          * The default clocks and voltages before the MC ucode
2272          * is loaded are not suffient for advanced operations.
2273          *
2274          * We can skip this check for TN, because there is no MC
2275          * ucode.
2276          */
2277         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2278                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2279                 return -EINVAL;
2280         }
2281
2282         return 0;
2283 }
2284
2285 void cayman_fini(struct radeon_device *rdev)
2286 {
2287         radeon_pm_fini(rdev);
2288         cayman_cp_fini(rdev);
2289         cayman_dma_fini(rdev);
2290         r600_irq_fini(rdev);
2291         if (rdev->flags & RADEON_IS_IGP)
2292                 sumo_rlc_fini(rdev);
2293         radeon_wb_fini(rdev);
2294         radeon_vm_manager_fini(rdev);
2295         radeon_ib_pool_fini(rdev);
2296         radeon_irq_kms_fini(rdev);
2297         uvd_v1_0_fini(rdev);
2298         radeon_uvd_fini(rdev);
2299         cayman_pcie_gart_fini(rdev);
2300         r600_vram_scratch_fini(rdev);
2301         radeon_gem_fini(rdev);
2302         radeon_fence_driver_fini(rdev);
2303         radeon_bo_fini(rdev);
2304         radeon_atombios_fini(rdev);
2305         kfree(rdev->bios);
2306         rdev->bios = NULL;
2307 }
2308
2309 /*
2310  * vm
2311  */
2312 int cayman_vm_init(struct radeon_device *rdev)
2313 {
2314         /* number of VMs */
2315         rdev->vm_manager.nvm = 8;
2316         /* base offset of vram pages */
2317         if (rdev->flags & RADEON_IS_IGP) {
2318                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2319                 tmp <<= 22;
2320                 rdev->vm_manager.vram_base_offset = tmp;
2321         } else
2322                 rdev->vm_manager.vram_base_offset = 0;
2323         return 0;
2324 }
2325
2326 void cayman_vm_fini(struct radeon_device *rdev)
2327 {
2328 }
2329
2330 /**
2331  * cayman_vm_decode_fault - print human readable fault info
2332  *
2333  * @rdev: radeon_device pointer
2334  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2335  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2336  *
2337  * Print human readable fault information (cayman/TN).
2338  */
2339 void cayman_vm_decode_fault(struct radeon_device *rdev,
2340                             u32 status, u32 addr)
2341 {
2342         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2343         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2344         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2345         char *block;
2346
2347         switch (mc_id) {
2348         case 32:
2349         case 16:
2350         case 96:
2351         case 80:
2352         case 160:
2353         case 144:
2354         case 224:
2355         case 208:
2356                 block = "CB";
2357                 break;
2358         case 33:
2359         case 17:
2360         case 97:
2361         case 81:
2362         case 161:
2363         case 145:
2364         case 225:
2365         case 209:
2366                 block = "CB_FMASK";
2367                 break;
2368         case 34:
2369         case 18:
2370         case 98:
2371         case 82:
2372         case 162:
2373         case 146:
2374         case 226:
2375         case 210:
2376                 block = "CB_CMASK";
2377                 break;
2378         case 35:
2379         case 19:
2380         case 99:
2381         case 83:
2382         case 163:
2383         case 147:
2384         case 227:
2385         case 211:
2386                 block = "CB_IMMED";
2387                 break;
2388         case 36:
2389         case 20:
2390         case 100:
2391         case 84:
2392         case 164:
2393         case 148:
2394         case 228:
2395         case 212:
2396                 block = "DB";
2397                 break;
2398         case 37:
2399         case 21:
2400         case 101:
2401         case 85:
2402         case 165:
2403         case 149:
2404         case 229:
2405         case 213:
2406                 block = "DB_HTILE";
2407                 break;
2408         case 38:
2409         case 22:
2410         case 102:
2411         case 86:
2412         case 166:
2413         case 150:
2414         case 230:
2415         case 214:
2416                 block = "SX";
2417                 break;
2418         case 39:
2419         case 23:
2420         case 103:
2421         case 87:
2422         case 167:
2423         case 151:
2424         case 231:
2425         case 215:
2426                 block = "DB_STEN";
2427                 break;
2428         case 40:
2429         case 24:
2430         case 104:
2431         case 88:
2432         case 232:
2433         case 216:
2434         case 168:
2435         case 152:
2436                 block = "TC_TFETCH";
2437                 break;
2438         case 41:
2439         case 25:
2440         case 105:
2441         case 89:
2442         case 233:
2443         case 217:
2444         case 169:
2445         case 153:
2446                 block = "TC_VFETCH";
2447                 break;
2448         case 42:
2449         case 26:
2450         case 106:
2451         case 90:
2452         case 234:
2453         case 218:
2454         case 170:
2455         case 154:
2456                 block = "VC";
2457                 break;
2458         case 112:
2459                 block = "CP";
2460                 break;
2461         case 113:
2462         case 114:
2463                 block = "SH";
2464                 break;
2465         case 115:
2466                 block = "VGT";
2467                 break;
2468         case 178:
2469                 block = "IH";
2470                 break;
2471         case 51:
2472                 block = "RLC";
2473                 break;
2474         case 55:
2475                 block = "DMA";
2476                 break;
2477         case 56:
2478                 block = "HDP";
2479                 break;
2480         default:
2481                 block = "unknown";
2482                 break;
2483         }
2484
2485         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2486                protections, vmid, addr,
2487                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2488                block, mc_id);
2489 }
2490
2491 /**
2492  * cayman_vm_flush - vm flush using the CP
2493  *
2494  * @rdev: radeon_device pointer
2495  *
2496  * Update the page table base and flush the VM TLB
2497  * using the CP (cayman-si).
2498  */
2499 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2500                      unsigned vm_id, uint64_t pd_addr)
2501 {
2502         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2503         radeon_ring_write(ring, pd_addr >> 12);
2504
2505         /* flush hdp cache */
2506         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2507         radeon_ring_write(ring, 0x1);
2508
2509         /* bits 0-7 are the VM contexts0-7 */
2510         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2511         radeon_ring_write(ring, 1 << vm_id);
2512
2513         /* wait for the invalidate to complete */
2514         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2515         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2516                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2517         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2518         radeon_ring_write(ring, 0);
2519         radeon_ring_write(ring, 0); /* ref */
2520         radeon_ring_write(ring, 0); /* mask */
2521         radeon_ring_write(ring, 0x20); /* poll interval */
2522
2523         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2524         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2525         radeon_ring_write(ring, 0x0);
2526 }