arch/arm/mach-rockchip/ddr_rk3126.c

   1 /*
   2  * arch/arm/mach-rk2928/ddr.c-- for ddr3&ddr2
   3  *
   4  * Function Driver for DDR controller
   5  *
   6  * Copyright (C) 2012 Fuzhou Rockchip Electronics Co.,Ltd
   7  * Author:
   8  * hcy@rock-chips.com
   9  * yk@rock-chips.com
  10  * typ@rock-chips.com
  11  *
  12  * v1.00
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/io.h>
  17 #include <linux/module.h>
  18 #include <linux/clk.h>
  19
  20 #include <asm/cacheflush.h>
  21 #include <asm/tlbflush.h>
  22 #include <linux/cpu.h>
  23 #include <dt-bindings/clock/ddr.h>
  24 #include <linux/rockchip/cpu_axi.h>
  25 #include <linux/rockchip/cru.h>
  26 #include <linux/rk_fb.h>
  27
  28 typedef uint32_t uint32;
  29
  30 #define DDR3_DDR2_DLL_DISABLE_FREQ    (300)     /* ¿ÅÁ£dll disableµÄÆµÂÊ*/
  31 #define DDR3_DDR2_ODT_DISABLE_FREQ    (333)     /*¿ÅÁ£odt disableµÄÆµÂÊ*/
  32 #define SR_IDLE                       (0x1)     /*unit:32*DDR clk cycle, and 0 for disable auto self-refresh*/
  33 #define PD_IDLE                       (0x40)    /*unit:DDR clk cycle, and 0 for disable auto power-down*/
  34 #define PHY_ODT_DISABLE_FREQ          (333)     /*¶¨ÒåÖ÷¿Ø¶Ëodt disableµÄÆµÂÊ*/
  35 #define PHY_DLL_DISABLE_FREQ          (266)     /*¶¨ÒåÖ÷¿Ø¶Ëdll bypassµÄÆµÂÊ*/
  36
  37 #define ddr_print(x...) printk("DDR DEBUG: " x)
  38
  39 #define SRAM_CODE_OFFSET        rockchip_sram_virt
  40 #define SRAM_SIZE               rockchip_sram_size
  41
  42 #ifdef CONFIG_FB_ROCKCHIP
  43 #define DDR_CHANGE_FREQ_IN_LCDC_VSYNC
  44 #endif
  45
  46 /*#define PHY_RX_PHASE_CAL*/
  47 #define PHY_DE_SKEW_STEP  (20)
  48 /***********************************
  49  * DDR3 define
  50  ***********************************/
  51 /*mr0 for ddr3*/
  52 #define DDR3_BL8          (0)
  53 #define DDR3_BC4_8        (1)
  54 #define DDR3_BC4          (2)
  55 #define DDR3_CL(n)        (((((n)-4)&0x7)<<4)|((((n)-4)&0x8)>>1))
  56 #define DDR3_WR(n)        (((n)&0x7)<<9)
  57 #define DDR3_DLL_RESET    (1<<8)
  58 #define DDR3_DLL_DeRESET  (0<<8)
  59
  60 /*mr1 for ddr3*/
  61 #define DDR3_DLL_ENABLE    (0)
  62 #define DDR3_DLL_DISABLE   (1)
  63 #define DDR3_MR1_AL(n)  (((n)&0x7)<<3)
  64
  65 #define DDR3_DS_40            (0)
  66 #define DDR3_DS_34            (1<<1)
  67 #define DDR3_Rtt_Nom_DIS      (0)
  68 #define DDR3_Rtt_Nom_60       (1<<2)
  69 #define DDR3_Rtt_Nom_120      (1<<6)
  70 #define DDR3_Rtt_Nom_40       ((1<<2)|(1<<6))
  71
  72 /*mr2 for ddr3*/
  73 #define DDR3_MR2_CWL(n) ((((n)-5)&0x7)<<3)
  74 #define DDR3_Rtt_WR_DIS       (0)
  75 #define DDR3_Rtt_WR_60        (1<<9)
  76 #define DDR3_Rtt_WR_120       (2<<9)
  77
  78 #define DDR_PLL_REFDIV  (1)
  79 #define FBDIV(n)        ((0xFFF<<16) | (n&0xfff))
  80 #define REFDIV(n)       ((0x3F<<16) | (n&0x3f))
  81 #define POSTDIV1(n)     ((0x7<<(12+16)) | ((n&0x7)<<12))
  82 #define POSTDIV2(n)     ((0x7<<(6+16)) | ((n&0x7)<<6))
  83
  84 #define PLL_LOCK_STATUS  (0x1<<10)
  85  /*CRU Registers updated*/
  86 typedef volatile struct tagCRU_STRUCT {
  87         uint32 CRU_PLL_CON[4][4];
  88         uint32 CRU_MODE_CON;
  89         uint32 CRU_CLKSEL_CON[35];
  90         uint32 CRU_CLKGATE_CON[11];     /*0xd0*/
  91         uint32 reserved1;       /*0xfc*/
  92         uint32 CRU_GLB_SRST_FST_VALUE;  /*0x100*/
  93         uint32 CRU_GLB_SRST_SND_VALUE;
  94         uint32 reserved2[2];
  95         uint32 CRU_SOFTRST_CON[9];      /*0x110*/
  96         uint32 CRU_MISC_CON;    /*0x134*/
  97         uint32 reserved3[2];
  98         uint32 CRU_GLB_CNT_TH;  /*0x140*/
  99         uint32 reserved4[3];
 100         uint32 CRU_GLB_RST_ST;  /*0x150*/
 101         uint32 reserved5[(0x1c0 - 0x154) / 4];
 102         uint32 CRU_SDMMC_CON[2];        /*0x1c0*/
 103         uint32 CRU_SDIO_CON[2];
 104         uint32 reserved6[2];
 105         uint32 CRU_EMMC_CON[2]; /*0x1d8*/
 106         uint32 reserved7[(0x1f0 - 0x1e0) / 4];
 107         uint32 CRU_PLL_PRG_EN;
 108 } CRU_REG, *pCRU_REG;
 109
 110 typedef struct tagGPIO_LH {
 111         uint32 GPIOL;
 112         uint32 GPIOH;
 113 } GPIO_LH_T;
 114
 115 typedef struct tagGPIO_IOMUX {
 116         uint32 GPIOA_IOMUX;
 117         uint32 GPIOB_IOMUX;
 118         uint32 GPIOC_IOMUX;
 119         uint32 GPIOD_IOMUX;
 120 } GPIO_IOMUX_T;
 121
 122 /********************************
 123 *GRF ¼Ä´æÆ÷ÖÐGRF_OS_REG1 ´æddr rank£¬typeµÈÐÅÏ¢
 124 *GRF_SOC_CON2¼Ä´æÆ÷ÖÐ¿ØÖÆc_sysreqÐÅºÅÏòpctl·¢ËÍ½øÈëlow power ÇëÇó
 125 *GRF_DDRC_STAT ¿É²éÑ¯pctlÊÇ·ñ½ÓÊÜÇëÇó ½øÈëlow power
 126 ********************************/
 127 /*REG FILE registers*/
 128 /*GRF_SOC_CON0*/
 129 #define DDR_MONITOR_EN  ((1<<(16+6))+(1<<6))
 130 #define DDR_MONITOR_DISB  ((1<<(16+6))+(0<<6))
 131
 132 /*GRF_SOC_STATUS0*/
 133 #define sys_pwr_idle     (1<<27)
 134 #define gpu_pwr_idle     (1<<26)
 135 #define vpu_pwr_idle     (1<<25)
 136 #define vio_pwr_idle     (1<<24)
 137 #define peri_pwr_idle    (1<<23)
 138 #define core_pwr_idle     (1<<22)
 139 /*GRF_SOC_CON2*/
 140 #define core_pwr_idlereq    (13)
 141 #define peri_pwr_idlereq    (12)
 142 #define vio_pwr_idlereq     (11)
 143 #define vpu_pwr_idlereq     (10)
 144 #define gpu_pwr_idlereq     (9)
 145 #define sys_pwr_idlereq     (8)
 146 #define GRF_DDR_LP_EN     (0x1<<(2+16))
 147 #define GRF_DDR_LP_DISB     ((0x1<<(2+16))|(0x1<<2))
 148
 149 /*grf updated*/
 150 typedef volatile struct tagREG_FILE {
 151         uint32 reserved0[(0xa8 - 0x0) / 4];
 152         GPIO_IOMUX_T GRF_GPIO_IOMUX[4]; /*0x00a8*/
 153         uint32 GRF_GPIO2C_IOMUX2;       /*0xe8*/
 154         uint32 GRF_CIF_IOMUX[2];
 155         uint32 reserved1[(0x100 - 0xf4) / 4];
 156         uint32 GRF_GPIO_DS;     /*0x100*/
 157         uint32 reserved2[(0x118 - 0x104) / 4];
 158         GPIO_LH_T GRF_GPIO_PULL[4];     /*0x118*/
 159         uint32 reserved3[1];
 160         uint32 GRF_ACODEC_CON;  /*0x13c*/
 161         uint32 GRF_SOC_CON[3];  /*0x140*/
 162         uint32 GRF_SOC_STATUS0;
 163         uint32 GRF_LVDS_CON0;   /*0x150*/
 164         uint32 reserved4[(0x15c - 0x154) / 4];
 165         uint32 GRF_DMAC_CON[3]; /*0x15c*/
 166         uint32 GRF_MAC_CON[2];
 167         uint32 GRF_TVE_CON;     /*0x170*/
 168         uint32 reserved5[(0x17c - 0x174) / 4];
 169         uint32 GRF_UOC0_CON0;   /*0x17c*/
 170         uint32 reserved6;
 171         uint32 GRF_UOC1_CON[5]; /*0x184*/
 172         uint32 reserved7;
 173         uint32 GRF_DDRC_STAT;   /*0x19c*/
 174         uint32 reserved8;
 175         uint32 GRF_SOC_STATUS1; /*0x1a4*/
 176         uint32 GRF_CPU_CON[4];
 177         uint32 reserved9[(0x1c0 - 0x1b8) / 4];
 178         uint32 GRF_CPU_STATUS[2];       /*0x1c0*/
 179         uint32 GRF_OS_REG[8];
 180         uint32 reserved10[(0x200 - 0x1e8) / 4];
 181         uint32 GRF_PVTM_CON[4]; /*0x200*/
 182         uint32 GRF_PVTM_STATUS[4];
 183         /*uint32 reserved10[(0x220-0x214)/4];*/
 184         uint32 GRF_DFI_WRNUM;   /*0X220*/
 185         uint32 GRF_DFI_RDNUM;
 186         uint32 GRF_DFI_ACTNUM;
 187         uint32 GRF_DFI_TIMERVAL;
 188         uint32 GRF_NIF_FIFO[4];
 189         uint32 reserved11[(0x280 - 0x240) / 4];
 190         uint32 GRF_USBPHY0_CON[8];      /*0x280*/
 191         uint32 GRF_USBPHY1_CON[8];
 192         uint32 GRF_UOC_STATUS0; /*0x2c0*/
 193         uint32 reserved12[(0x300 - 0x2c4) / 4];
 194         uint32 GRF_CHIP_TAG;
 195         uint32 GRF_SDMMC_DET_CNT;
 196         uint32 reserved13[(0x37c - 0x308) / 4];
 197         uint32 GRF_EFUSE_PRG_EN;
 198 } REG_FILE, *pREG_FILE;
 199
 200 /*SCTL*/
 201 #define INIT_STATE                     (0)
 202 #define CFG_STATE                      (1)
 203 #define GO_STATE                       (2)
 204 #define SLEEP_STATE                    (3)
 205 #define WAKEUP_STATE                   (4)
 206
 207 /*STAT*/
 208 #define Init_mem                       (0)
 209 #define Config                         (1)
 210 #define Config_req                     (2)
 211 #define Access                         (3)
 212 #define Access_req                     (4)
 213 #define Low_power                      (5)
 214 #define Low_power_entry_req            (6)
 215 #define Low_power_exit_req             (7)
 216
 217 /*MCFG*/
 218 #define mddr_lpddr2_clk_stop_idle(n)   ((n)<<24)
 219 #define pd_idle(n)                     ((n)<<8)
 220 #define mddr_en                        (2<<22)
 221 #define lpddr2_en                      (3<<22)
 222 #define ddr2_en                        (0<<5)
 223 #define ddr3_en                        (1<<5)
 224 #define lpddr2_s2                      (0<<6)
 225 #define lpddr2_s4                      (1<<6)
 226 #define mddr_lpddr2_bl_2               (0<<20)
 227 #define mddr_lpddr2_bl_4               (1<<20)
 228 #define mddr_lpddr2_bl_8               (2<<20)
 229 #define mddr_lpddr2_bl_16              (3<<20)
 230 #define ddr2_ddr3_bl_4                 (0)
 231 #define ddr2_ddr3_bl_8                 (1)
 232 #define tfaw_cfg(n)                    (((n)-4)<<18)
 233 #define pd_exit_slow                   (0<<17)
 234 #define pd_exit_fast                   (1<<17)
 235 #define pd_type(n)                     ((n)<<16)
 236 #define two_t_en(n)                    ((n)<<3)
 237 #define bl8int_en(n)                   ((n)<<2)
 238 #define cke_or_en(n)                   ((n)<<1)
 239
 240 /*POWCTL*/
 241 #define power_up_start                 (1<<0)
 242
 243 /*POWSTAT*/
 244 #define power_up_done                  (1<<0)
 245
 246 /*DFISTSTAT0*/
 247 #define dfi_init_complete              (1<<0)
 248
 249 /*CMDTSTAT*/
 250 #define cmd_tstat                      (1<<0)
 251
 252 /*CMDTSTATEN*/
 253 #define cmd_tstat_en                   (1<<1)
 254
 255 /*MCMD*/
 256 #define Deselect_cmd                   (0)
 257 #define PREA_cmd                       (1)
 258 #define REF_cmd                        (2)
 259 #define MRS_cmd                        (3)
 260 #define ZQCS_cmd                       (4)
 261 #define ZQCL_cmd                       (5)
 262 #define RSTL_cmd                       (6)
 263 #define MRR_cmd                        (8)
 264 #define DPDE_cmd                       (9)
 265
 266 #define lpddr2_op(n)                   ((n)<<12)
 267 #define lpddr2_ma(n)                   ((n)<<4)
 268
 269 #define bank_addr(n)                   ((n)<<17)
 270 #define cmd_addr(n)                    ((n)<<4)
 271
 272 #define start_cmd                      (1u<<31)
 273
 274 typedef union STAT_Tag {
 275         uint32 d32;
 276         struct {
 277                 unsigned ctl_stat:3;
 278                 unsigned reserved3:1;
 279                 unsigned lp_trig:3;
 280                 unsigned reserved7_31:25;
 281         } b;
 282 } STAT_T;
 283
 284 typedef union SCFG_Tag {
 285         uint32 d32;
 286         struct {
 287                 unsigned hw_low_power_en:1;
 288                 unsigned reserved1_5:5;
 289                 unsigned nfifo_nif1_dis:1;
 290                 unsigned reserved7:1;
 291                 unsigned bbflags_timing:4;
 292                 unsigned reserved12_31:20;
 293         } b;
 294 } SCFG_T;
 295
 296 /* DDR Controller register struct */
 297 typedef volatile struct DDR_REG_Tag {
 298         /*Operational State, Control, and Status Registers*/
 299         SCFG_T SCFG;            /*State Configuration Register*/
 300         volatile uint32 SCTL;   /*State Control Register*/
 301         STAT_T STAT;            /*State Status Register*/
 302         volatile uint32 INTRSTAT;       /*Interrupt Status Register*/
 303         uint32 reserved0[(0x40 - 0x10) / 4];
 304         /*Initailization Control and Status Registers*/
 305         volatile uint32 MCMD;   /*Memory Command Register*/
 306         volatile uint32 POWCTL; /*Power Up Control Registers*/
 307         volatile uint32 POWSTAT;        /*Power Up Status Register*/
 308         volatile uint32 CMDTSTAT;       /*Command Timing Status Register*/
 309         volatile uint32 CMDTSTATEN;     /*Command Timing Status Enable Register*/
 310         uint32 reserved1[(0x60 - 0x54) / 4];
 311         volatile uint32 MRRCFG0;        /*MRR Configuration 0 Register*/
 312         volatile uint32 MRRSTAT0;       /*MRR Status 0 Register*/
 313         volatile uint32 MRRSTAT1;       /*MRR Status 1 Register*/
 314         uint32 reserved2[(0x7c - 0x6c) / 4];
 315         /*Memory Control and Status Registers*/
 316         volatile uint32 MCFG1;  /*Memory Configuration 1 Register*/
 317         volatile uint32 MCFG;   /*Memory Configuration Register*/
 318         volatile uint32 PPCFG;  /*Partially Populated Memories Configuration Register*/
 319         volatile uint32 MSTAT;  /*Memory Status Register*/
 320         volatile uint32 LPDDR2ZQCFG;    /*LPDDR2 ZQ Configuration Register*/
 321         uint32 reserved3;
 322         /*DTU Control and Status Registers*/
 323         volatile uint32 DTUPDES;        /*DTU Status Register*/
 324         volatile uint32 DTUNA;  /*DTU Number of Random Addresses Created Register*/
 325         volatile uint32 DTUNE;  /*DTU Number of Errors Register*/
 326         volatile uint32 DTUPRD0;        /*DTU Parallel Read 0*/
 327         volatile uint32 DTUPRD1;        /*DTU Parallel Read 1*/
 328         volatile uint32 DTUPRD2;        /*DTU Parallel Read 2*/
 329         volatile uint32 DTUPRD3;        /*DTU Parallel Read 3*/
 330         volatile uint32 DTUAWDT;        /*DTU Address Width*/
 331         uint32 reserved4[(0xc0 - 0xb4) / 4];
 332         /*Memory Timing Registers*/
 333         volatile uint32 TOGCNT1U;       /*Toggle Counter 1U Register*/
 334         volatile uint32 TINIT;  /*t_init Timing Register*/
 335         volatile uint32 TRSTH;  /*Reset High Time Register*/
 336         volatile uint32 TOGCNT100N;     /*Toggle Counter 100N Register*/
 337         volatile uint32 TREFI;  /*t_refi Timing Register*/
 338         volatile uint32 TMRD;   /*t_mrd Timing Register*/
 339         volatile uint32 TRFC;   /*t_rfc Timing Register*/
 340         volatile uint32 TRP;    /*t_rp Timing Register*/
 341         volatile uint32 TRTW;   /*t_rtw Timing Register*/
 342         volatile uint32 TAL;    /*AL Latency Register*/
 343         volatile uint32 TCL;    /*CL Timing Register*/
 344         volatile uint32 TCWL;   /*CWL Register*/
 345         volatile uint32 TRAS;   /*t_ras Timing Register*/
 346         volatile uint32 TRC;    /*t_rc Timing Register*/
 347         volatile uint32 TRCD;   /*t_rcd Timing Register*/
 348         volatile uint32 TRRD;   /*t_rrd Timing Register*/
 349         volatile uint32 TRTP;   /*t_rtp Timing Register*/
 350         volatile uint32 TWR;    /*t_wr Timing Register*/
 351         volatile uint32 TWTR;   /*t_wtr Timing Register*/
 352         volatile uint32 TEXSR;  /*t_exsr Timing Register*/
 353         volatile uint32 TXP;    /*t_xp Timing Register*/
 354         volatile uint32 TXPDLL; /*t_xpdll Timing Register*/
 355         volatile uint32 TZQCS;  /*t_zqcs Timing Register*/
 356         volatile uint32 TZQCSI; /*t_zqcsi Timing Register*/
 357         volatile uint32 TDQS;   /*t_dqs Timing Register*/
 358         volatile uint32 TCKSRE; /*t_cksre Timing Register*/
 359         volatile uint32 TCKSRX; /*t_cksrx Timing Register*/
 360         volatile uint32 TCKE;   /*t_cke Timing Register*/
 361         volatile uint32 TMOD;   /*t_mod Timing Register*/
 362         volatile uint32 TRSTL;  /*Reset Low Timing Register*/
 363         volatile uint32 TZQCL;  /*t_zqcl Timing Register*/
 364         volatile uint32 TMRR;   /*t_mrr Timing Register*/
 365         volatile uint32 TCKESR; /*t_ckesr Timing Register*/
 366         volatile uint32 TDPD;   /*t_dpd Timing Register*/
 367         uint32 reserved5[(0x180 - 0x148) / 4];
 368         /*ECC Configuration, Control, and Status Registers*/
 369         volatile uint32 ECCCFG; /*ECC Configuration Register*/
 370         volatile uint32 ECCTST; /*ECC Test Register*/
 371         volatile uint32 ECCCLR; /*ECC Clear Register*/
 372         volatile uint32 ECCLOG; /*ECC Log Register*/
 373         uint32 reserved6[(0x200 - 0x190) / 4];
 374         /*DTU Control and Status Registers*/
 375         volatile uint32 DTUWACTL;       /*DTU Write Address Control Register*/
 376         volatile uint32 DTURACTL;       /*DTU Read Address Control Register*/
 377         volatile uint32 DTUCFG; /*DTU Configuration Control Register*/
 378         volatile uint32 DTUECTL;        /*DTU Execute Control Register*/
 379         volatile uint32 DTUWD0; /*DTU Write Data 0*/
 380         volatile uint32 DTUWD1; /*DTU Write Data 1*/
 381         volatile uint32 DTUWD2; /*DTU Write Data 2*/
 382         volatile uint32 DTUWD3; /*DTU Write Data 3*/
 383         volatile uint32 DTUWDM; /*DTU Write Data Mask*/
 384         volatile uint32 DTURD0; /*DTU Read Data 0*/
 385         volatile uint32 DTURD1; /*DTU Read Data 1*/
 386         volatile uint32 DTURD2; /*DTU Read Data 2*/
 387         volatile uint32 DTURD3; /*DTU Read Data 3*/
 388         volatile uint32 DTULFSRWD;      /*DTU LFSR Seed for Write Data Generation*/
 389         volatile uint32 DTULFSRRD;      /*DTU LFSR Seed for Read Data Generation*/
 390         volatile uint32 DTUEAF; /*DTU Error Address FIFO*/
 391         /*DFI Control Registers*/
 392         volatile uint32 DFITCTRLDELAY;  /*DFI tctrl_delay Register*/
 393         volatile uint32 DFIODTCFG;      /*DFI ODT Configuration Register*/
 394         volatile uint32 DFIODTCFG1;     /*DFI ODT Configuration 1 Register*/
 395         volatile uint32 DFIODTRANKMAP;  /*DFI ODT Rank Mapping Register*/
 396         /*DFI Write Data Registers*/
 397         volatile uint32 DFITPHYWRDATA;  /*DFI tphy_wrdata Register*/
 398         volatile uint32 DFITPHYWRLAT;   /*DFI tphy_wrlat Register*/
 399         uint32 reserved7[(0x260 - 0x258) / 4];
 400         volatile uint32 DFITRDDATAEN;   /*DFI trddata_en Register*/
 401         volatile uint32 DFITPHYRDLAT;   /*DFI tphy_rddata Register*/
 402         uint32 reserved8[(0x270 - 0x268) / 4];
 403         /*DFI Update Registers*/
 404         volatile uint32 DFITPHYUPDTYPE0;        /*DFI tphyupd_type0 Register*/
 405         volatile uint32 DFITPHYUPDTYPE1;        /*DFI tphyupd_type1 Register*/
 406         volatile uint32 DFITPHYUPDTYPE2;        /*DFI tphyupd_type2 Register*/
 407         volatile uint32 DFITPHYUPDTYPE3;        /*DFI tphyupd_type3 Register*/
 408         volatile uint32 DFITCTRLUPDMIN; /*DFI tctrlupd_min Register*/
 409         volatile uint32 DFITCTRLUPDMAX; /*DFI tctrlupd_max Register*/
 410         volatile uint32 DFITCTRLUPDDLY; /*DFI tctrlupd_dly Register*/
 411         uint32 reserved9;
 412         volatile uint32 DFIUPDCFG;      /*DFI Update Configuration Register*/
 413         volatile uint32 DFITREFMSKI;    /*DFI Masked Refresh Interval Register*/
 414         volatile uint32 DFITCTRLUPDI;   /*DFI tctrlupd_interval Register*/
 415         uint32 reserved10[(0x2ac - 0x29c) / 4];
 416         volatile uint32 DFITRCFG0;      /*DFI Training Configuration 0 Register*/
 417         volatile uint32 DFITRSTAT0;     /*DFI Training Status 0 Register*/
 418         volatile uint32 DFITRWRLVLEN;   /*DFI Training dfi_wrlvl_en Register*/
 419         volatile uint32 DFITRRDLVLEN;   /*DFI Training dfi_rdlvl_en Register*/
 420         volatile uint32 DFITRRDLVLGATEEN;       /*DFI Training dfi_rdlvl_gate_en Register*/
 421         /*DFI Status Registers*/
 422         volatile uint32 DFISTSTAT0;     /*DFI Status Status 0 Register*/
 423         volatile uint32 DFISTCFG0;      /*DFI Status Configuration 0 Register*/
 424         volatile uint32 DFISTCFG1;      /*DFI Status configuration 1 Register*/
 425         uint32 reserved11;
 426         volatile uint32 DFITDRAMCLKEN;  /*DFI tdram_clk_enalbe Register*/
 427         volatile uint32 DFITDRAMCLKDIS; /*DFI tdram_clk_disalbe Register*/
 428         volatile uint32 DFISTCFG2;      /*DFI Status configuration 2 Register*/
 429         volatile uint32 DFISTPARCLR;    /*DFI Status Parity Clear Register*/
 430         volatile uint32 DFISTPARLOG;    /*DFI Status Parity Log Register*/
 431         uint32 reserved12[(0x2f0 - 0x2e4) / 4];
 432         /*DFI Low Power Registers*/
 433         volatile uint32 DFILPCFG0;      /*DFI Low Power Configuration 0 Register*/
 434         uint32 reserved13[(0x300 - 0x2f4) / 4];
 435         /*DFI Training 2 Registers*/
 436         volatile uint32 DFITRWRLVLRESP0;        /*DFI Training dif_wrlvl_resp Status 0 Register*/
 437         volatile uint32 DFITRWRLVLRESP1;        /*DFI Training dif_wrlvl_resp Status 1 Register*/
 438         volatile uint32 DFITRWRLVLRESP2;        /*DFI Training dif_wrlvl_resp Status 2 Register*/
 439         volatile uint32 DFITRRDLVLRESP0;        /*DFI Training dif_rdlvl_resp Status 0 Register*/
 440         volatile uint32 DFITRRDLVLRESP1;        /*DFI Training dif_rdlvl_resp Status 1 Register*/
 441         volatile uint32 DFITRRDLVLRESP2;        /*DFI Training dif_rdlvl_resp Status 2 Register*/
 442         volatile uint32 DFITRWRLVLDELAY0;       /*DFI Training dif_wrlvl_delay Configuration 0 Register*/
 443         volatile uint32 DFITRWRLVLDELAY1;       /*DFI Training dif_wrlvl_delay Configuration 1 Register*/
 444         volatile uint32 DFITRWRLVLDELAY2;       /*DFI Training dif_wrlvl_delay Configuration 2 Register*/
 445         volatile uint32 DFITRRDLVLDELAY0;       /*DFI Training dif_rdlvl_delay Configuration 0 Register*/
 446         volatile uint32 DFITRRDLVLDELAY1;       /*DFI Training dif_rdlvl_delay Configuration 1 Register*/
 447         volatile uint32 DFITRRDLVLDELAY2;       /*DFI Training dif_rdlvl_delay Configuration 2 Register*/
 448         volatile uint32 DFITRRDLVLGATEDELAY0;   /*DFI Training dif_rdlvl_gate_delay Configuration 0 Register*/
 449         volatile uint32 DFITRRDLVLGATEDELAY1;   /*DFI Training dif_rdlvl_gate_delay Configuration 1 Register*/
 450         volatile uint32 DFITRRDLVLGATEDELAY2;   /*DFI Training dif_rdlvl_gate_delay Configuration 2 Register*/
 451         volatile uint32 DFITRCMD;       /*DFI Training Command Register*/
 452         uint32 reserved14[(0x3f8 - 0x340) / 4];
 453         /*IP Status Registers*/
 454         volatile uint32 IPVR;   /*IP Version Register*/
 455         volatile uint32 IPTR;   /*IP Type Register*/
 456 } DDR_REG_T, *pDDR_REG_T;
 457
 458 /*PHY_REG2*/
 459 #define PHY_AUTO_CALIBRATION (1<<0)
 460 #define PHY_SW_CALIBRATION   (1<<1)
 461 /*PHY_REG1*/
 462 #define PHY_DDR2             (1)
 463 #define PHY_DDR3             (0)
 464 #define PHY_LPDDR2           (2)
 465 #define PHY_Burst8           (1<<2)
 466
 467 #define PHY_RON_DISABLE     (0)
 468 #define PHY_RON_309ohm      (1)
 469 #define PHY_RON_155ohm      (2)
 470 #define PHY_RON_103ohm      (3)
 471 #define PHY_RON_77ohm       (4)
 472 #define PHY_RON_63ohm       (5)
 473 #define PHY_RON_52ohm       (6)
 474 #define PHY_RON_45ohm       (7)
 475 /*#define PHY_RON_77ohm       (8)*/
 476 #define PHY_RON_62ohm       (9)
 477 /*#define PHY_RON_52ohm       (10)*/
 478 #define PHY_RON_44ohm       (11)
 479 #define PHY_RON_39ohm       (12)
 480 #define PHY_RON_34ohm       (13)
 481 #define PHY_RON_31ohm       (14)
 482 #define PHY_RON_28ohm       (15)
 483
 484 #define PHY_RTT_DISABLE     (0)
 485 #define PHY_RTT_816ohm      (1)
 486 #define PHY_RTT_431ohm      (2)
 487 #define PHY_RTT_287ohm      (3)
 488 #define PHY_RTT_216ohm      (4)
 489 #define PHY_RTT_172ohm      (5)
 490 #define PHY_RTT_145ohm      (6)
 491 #define PHY_RTT_124ohm      (7)
 492 #define PHY_RTT_215ohm      (8)
 493 /*#define PHY_RTT_172ohm      (9)*/
 494 #define PHY_RTT_144ohm      (10)
 495 #define PHY_RTT_123ohm      (11)
 496 #define PHY_RTT_108ohm      (12)
 497 #define PHY_RTT_96ohm       (13)
 498 #define PHY_RTT_86ohm       (14)
 499 #define PHY_RTT_78ohm       (15)
 500
 501 #define PHY_DRV_ODT_SET(n) ((n<<4)|n)
 502
 503 /* DDR PHY register struct  updated */
 504 typedef volatile struct DDRPHY_REG_Tag {
 505         volatile uint32 PHY_REG0;       /*PHY soft reset Register*/
 506         volatile uint32 PHY_REG1;       /*phy working mode, burst length*/
 507         volatile uint32 PHY_REG2;       /*PHY DQS squelch calibration Register*/
 508         volatile uint32 PHY_REG3;       /*channel A read odt delay*/
 509         volatile uint32 PHY_REG4;       /*channel B read odt dleay*/
 510         uint32 reserved0[(0x2c - 0x14) / 4];
 511         volatile uint32 PHY_REGb;       /*cl,al*/
 512         volatile uint32 PHY_REGc;       /*CWL set register*/
 513         uint32 reserved1[(0x44 - 0x34) / 4];
 514         volatile uint32 PHY_REG11;      /*cmd drv*/
 515         volatile uint32 PHY_REG12;      /*cmd weak pull up*/
 516         volatile uint32 PHY_REG13;      /*cmd dll delay*/
 517         volatile uint32 PHY_REG14;      /*CK dll delay*/
 518         uint32 reserved2;       /*0x54*/
 519         volatile uint32 PHY_REG16;      /*/CK drv*/
 520         uint32 reserved3[(0x80 - 0x5c) / 4];
 521         volatile uint32 PHY_REG20;      /*left channel a drv*/
 522         volatile uint32 PHY_REG21;      /*left channel a odt*/
 523         uint32 reserved4[(0x98 - 0x88) / 4];
 524         volatile uint32 PHY_REG26;      /*left channel a dq write dll*/
 525         volatile uint32 PHY_REG27;      /*left channel a dqs write dll*/
 526         volatile uint32 PHY_REG28;      /*left channel a dqs read dll*/
 527         uint32 reserved5[(0xc0 - 0xa4) / 4];
 528         volatile uint32 PHY_REG30;      /*right channel a drv*/
 529         volatile uint32 PHY_REG31;      /*right channel a odt*/
 530         uint32 reserved6[(0xd8 - 0xc8) / 4];
 531         volatile uint32 PHY_REG36;      /*right channel a dq write dll*/
 532         volatile uint32 PHY_REG37;      /*right channel a dqs write dll*/
 533         volatile uint32 PHY_REG38;      /*right channel a dqs read dll*/
 534         uint32 reserved7[(0x100 - 0xe4) / 4];
 535         volatile uint32 PHY_REG40;      /*left channel b drv*/
 536         volatile uint32 PHY_REG41;      /*left channel b odt*/
 537         uint32 reserved8[(0x118 - 0x108) / 4];
 538         volatile uint32 PHY_REG46;      /*left channel b dq write dll*/
 539         volatile uint32 PHY_REG47;      /*left channel b dqs write dll*/
 540         volatile uint32 PHY_REG48;      /*left channel b dqs read dll*/
 541         uint32 reserved9[(0x140 - 0x124) / 4];
 542         volatile uint32 PHY_REG50;      /*right channel b drv*/
 543         volatile uint32 PHY_REG51;      /*right channel b odt*/
 544         uint32 reserved10[(0x158 - 0x148) / 4];
 545         volatile uint32 PHY_REG56;      /*right channel b dq write dll*/
 546         volatile uint32 PHY_REG57;      /*right channel b dqs write dll*/
 547         volatile uint32 PHY_REG58;      /*right channel b dqs read dll*/
 548         uint32 reserved11[(0x290 - 0x164) / 4];
 549         volatile uint32 PHY_REGDLL;     /*dll bypass switch reg*/
 550         uint32 reserved12[(0x2c0 - 0x294) / 4];
 551         volatile uint32 PHY_REG_skew[(0x3b0 - 0x2c0) / 4];      /*de-skew*/
 552         uint32 reserved13[(0x3e8 - 0x3b0) / 4];
 553         volatile uint32 PHY_REGfa;      /*idqs*/
 554         volatile uint32 PHY_REGfb;      /* left channel a calibration result*/
 555         volatile uint32 PHY_REGfc;      /* right channel a calibration result*/
 556         volatile uint32 PHY_REGfd;      /*left channel b calibration result*/
 557         volatile uint32 PHY_REGfe;      /* right channel b calibration result*/
 558         volatile uint32 PHY_REGff;      /*calibrationg done*/
 559 } DDRPHY_REG_T, *pDDRPHY_REG_T;
 560
 561 #define pCRU_Reg               ((pCRU_REG)RK_CRU_VIRT)
 562 #define pGRF_Reg               ((pREG_FILE)RK_GRF_VIRT)
 563 #define pDDR_Reg               ((pDDR_REG_T)RK_DDR_VIRT)
 564 #define pPHY_Reg               ((pDDRPHY_REG_T)(RK_DDR_VIRT+RK3036_DDR_PCTL_SIZE))
 565 #define SysSrv_DdrTiming       (RK_CPU_AXI_BUS_VIRT+0xc)
 566 #define PMU_PWEDN_ST            (RK_PMU_VIRT + 0x8)
 567 #define READ_CS_INFO()   ((((pGRF_Reg->GRF_OS_REG[1])>>11)&0x1)+1)
 568 #define READ_COL_INFO()  (9+(((pGRF_Reg->GRF_OS_REG[1])>>9)&0x3))
 569 #define READ_BK_INFO()   (3-(((pGRF_Reg->GRF_OS_REG[1])>>8)&0x1))
 570 #define READ_CS0_ROW_INFO()  (13+(((pGRF_Reg->GRF_OS_REG[1])>>6)&0x3))
 571 #define READ_CS1_ROW_INFO()  (13+(((pGRF_Reg->GRF_OS_REG[1])>>4)&0x3))
 572 #define READ_BW_INFO()   (2>>(((pGRF_Reg->GRF_OS_REG[1])&0xc)>>2))      /*´úÂëÖÐ 0->8bit 1->16bit 2->32bit  ÓëgrfÖÐ¶¨ÒåÏà·´*/
 573 #define READ_DIE_BW_INFO()   (2>>((pGRF_Reg->GRF_OS_REG[1])&0x3))
 574
 575 /***********************************
 576  * LPDDR2 define
 577  ***********************************/
 578 /*MR0 (Device Information)*/
 579 #define  LPDDR2_DAI    (0x1)    /* 0:DAI complete, 1:DAI still in progress*/
 580 #define  LPDDR2_DI     (0x1<<1) /* 0:S2 or S4 SDRAM, 1:NVM*/
 581 #define  LPDDR2_DNVI   (0x1<<2) /* 0:DNV not supported, 1:DNV supported*/
 582 #define  LPDDR2_RZQI   (0x3<<3) /*00:RZQ self test not supported, 01:ZQ-pin may connect to VDDCA or float*/
 583                                     /*10:ZQ-pin may short to GND.     11:ZQ-pin self test completed, no error condition detected.*/
 584
 585 /*MR1 (Device Feature)*/
 586 #define LPDDR2_BL4     (0x2)
 587 #define LPDDR2_BL8     (0x3)
 588 #define LPDDR2_BL16    (0x4)
 589 #define LPDDR2_nWR(n)  (((n)-2)<<5)
 590
 591 /*MR2 (Device Feature 2)*/
 592 #define LPDDR2_RL3_WL1  (0x1)
 593 #define LPDDR2_RL4_WL2  (0x2)
 594 #define LPDDR2_RL5_WL2  (0x3)
 595 #define LPDDR2_RL6_WL3  (0x4)
 596 #define LPDDR2_RL7_WL4  (0x5)
 597 #define LPDDR2_RL8_WL4  (0x6)
 598
 599 /*MR3 (IO Configuration 1)*/
 600 #define LPDDR2_DS_34    (0x1)
 601 #define LPDDR2_DS_40    (0x2)
 602 #define LPDDR2_DS_48    (0x3)
 603 #define LPDDR2_DS_60    (0x4)
 604 #define LPDDR2_DS_80    (0x6)
 605 #define LPDDR2_DS_120   (0x7)   /*optional*/
 606
 607 /*MR4 (Device Temperature)*/
 608 #define LPDDR2_tREF_MASK (0x7)
 609 #define LPDDR2_4_tREF    (0x1)
 610 #define LPDDR2_2_tREF    (0x2)
 611 #define LPDDR2_1_tREF    (0x3)
 612 #define LPDDR2_025_tREF  (0x5)
 613 #define LPDDR2_025_tREF_DERATE    (0x6)
 614
 615 #define LPDDR2_TUF       (0x1<<7)
 616
 617 /*MR8 (Basic configuration 4)*/
 618 #define LPDDR2_S4        (0x0)
 619 #define LPDDR2_S2        (0x1)
 620 #define LPDDR2_N         (0x2)
 621 #define LPDDR2_Density(mr8)  (8<<(((mr8)>>2)&0xf))      /*Unit:MB*/
 622 #define LPDDR2_IO_Width(mr8) (32>>(((mr8)>>6)&0x3))
 623
 624 /*MR10 (Calibration)*/
 625 #define LPDDR2_ZQINIT   (0xFF)
 626 #define LPDDR2_ZQCL     (0xAB)
 627 #define LPDDR2_ZQCS     (0x56)
 628 #define LPDDR2_ZQRESET  (0xC3)
 629
 630 /*MR16 (PASR Bank Mask)*/
 631 /*S2 SDRAM Only*/
 632 #define LPDDR2_PASR_Full (0x0)
 633 #define LPDDR2_PASR_1_2  (0x1)
 634 #define LPDDR2_PASR_1_4  (0x2)
 635 #define LPDDR2_PASR_1_8  (0x3)
 636
 637 typedef enum PLL_ID_Tag {
 638         APLL = 0,
 639         DPLL,
 640         CPLL,
 641         GPLL,
 642         PLL_MAX
 643 } PLL_ID;
 644
 645 typedef enum DRAM_TYPE_Tag {
 646         LPDDR = 0,
 647         DDR,
 648         DDR2,
 649         DDR3,
 650         LPDDR2S2,
 651         LPDDR2,
 652
 653         DRAM_MAX
 654 } DRAM_TYPE;
 655
 656 struct ddr_freq_t {
 657         unsigned long screen_ft_us;
 658         unsigned long long t0;
 659         unsigned long long t1;
 660         unsigned long t2;
 661 };
 662
 663 typedef struct PCTRL_TIMING_Tag {
 664         uint32 ddrFreq;
 665         /*Memory Timing Registers*/
 666         uint32 togcnt1u;        /*Toggle Counter 1U Register*/
 667         uint32 tinit;           /*t_init Timing Register*/
 668         uint32 trsth;           /*Reset High Time Register*/
 669         uint32 togcnt100n;      /*Toggle Counter 100N Register*/
 670         uint32 trefi;           /*t_refi Timing Register*/
 671         uint32 tmrd;            /*t_mrd Timing Register*/
 672         uint32 trfc;            /*t_rfc Timing Register*/
 673         uint32 trp;                 /*t_rp Timing Register*/
 674         uint32 trtw;            /*t_rtw Timing Register*/
 675         uint32 tal;                 /*AL Latency Register*/
 676         uint32 tcl;                 /*CL Timing Register*/
 677         uint32 tcwl;            /*CWL Register*/
 678         uint32 tras;            /*t_ras Timing Register*/
 679         uint32 trc;                 /*t_rc Timing Register*/
 680         uint32 trcd;            /*t_rcd Timing Register*/
 681         uint32 trrd;            /*t_rrd Timing Register*/
 682         uint32 trtp;            /*t_rtp Timing Register*/
 683         uint32 twr;                 /*t_wr Timing Register*/
 684         uint32 twtr;            /*t_wtr Timing Register*/
 685         uint32 texsr;           /*t_exsr Timing Register*/
 686         uint32 txp;                 /*t_xp Timing Register*/
 687         uint32 txpdll;          /*t_xpdll Timing Register*/
 688         uint32 tzqcs;           /*t_zqcs Timing Register*/
 689         uint32 tzqcsi;          /*t_zqcsi Timing Register*/
 690         uint32 tdqs;            /*t_dqs Timing Register*/
 691         uint32 tcksre;          /*t_cksre Timing Register*/
 692         uint32 tcksrx;          /*t_cksrx Timing Register*/
 693         uint32 tcke;            /*t_cke Timing Register*/
 694         uint32 tmod;            /*t_mod Timing Register*/
 695         uint32 trstl;           /*Reset Low Timing Register*/
 696         uint32 tzqcl;           /*t_zqcl Timing Register*/
 697         uint32 tmrr;            /*t_mrr Timing Register*/
 698         uint32 tckesr;          /*t_ckesr Timing Register*/
 699         uint32 tdpd;            /*t_dpd Timing Register*/
 700 } PCTL_TIMING_T;
 701
 702 struct ddr_change_freq_sram_param {
 703         uint32 freq;
 704         uint32 freq_slew;
 705 };
 706
 707 typedef union NOC_TIMING_Tag {
 708         uint32 d32;
 709         struct {
 710                 unsigned ActToAct:6;
 711                 unsigned RdToMiss:6;
 712                 unsigned WrToMiss:6;
 713                 unsigned BurstLen:3;
 714                 unsigned RdToWr:5;
 715                 unsigned WrToRd:5;
 716                 unsigned BwRatio:1;
 717         } b;
 718 } NOC_TIMING_T;
 719
 720 typedef struct BACKUP_REG_Tag {
 721         PCTL_TIMING_T pctl_timing;
 722         NOC_TIMING_T noc_timing;
 723         uint32 ddrMR[4];
 724         uint32 mem_type;
 725         uint32 ddr_speed_bin;
 726         uint32 ddr_capability_per_die;
 727 } BACKUP_REG_T;
 728
 729 BACKUP_REG_T DEFINE_PIE_DATA(ddr_reg);
 730 static BACKUP_REG_T *p_ddr_reg;
 731
 732 uint32 DEFINE_PIE_DATA(ddr_freq);
 733 static uint32 *p_ddr_freq;
 734 uint32 DEFINE_PIE_DATA(ddr_sr_idle);
 735 uint32 DEFINE_PIE_DATA(ddr_dll_status); /* ¼ÇÂ¼ddr dllµÄ×´Ì¬£¬ÔÚselfrefresh exitÊ±Ñ¡ÔñÊÇ·ñ½øÐÐdll reset*/
 736
 737 static const uint32_t ddr3_cl_cwl[22][4] = {
 738 /*   0~330           330~400         400~533        speed
 739 * tCK  >3             2.5~3          1.875~2.5     1.875~1.5
 740 *    cl<<16, cwl    cl<<16, cwl     cl<<16, cwl              */
 741         {((5 << 16) | 5), ((5 << 16) | 5), 0, 0},       /*DDR3_800D*/
 742         {((5 << 16) | 5), ((6 << 16) | 5), 0, 0},       /*DDR3_800E*/
 743
 744         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), 0}, /*DDR3_1066E*/
 745         {((5 << 16) | 5), ((6 << 16) | 5), ((7 << 16) | 6), 0}, /*DDR3_1066F*/
 746         {((5 << 16) | 5), ((6 << 16) | 5), ((8 << 16) | 6), 0}, /*DDR3_1066G*/
 747
 748         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((7 << 16) | 7)},   /*DDR3_1333F*/
 749         {((5 << 16) | 5), ((5 << 16) | 5), ((7 << 16) | 6), ((8 << 16) | 7)},   /*DDR3_1333G*/
 750         {((5 << 16) | 5), ((6 << 16) | 5), ((7 << 16) | 6), ((9 << 16) | 7)},   /*DDR3_1333H*/
 751         {((5 << 16) | 5), ((6 << 16) | 5), ((8 << 16) | 6), ((10 << 16) | 7)},  /*DDR3_1333J*/
 752
 753         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((7 << 16) | 7)},   /*DDR3_1600G*/
 754         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((8 << 16) | 7)},   /*DDR3_1600H*/
 755         {((5 << 16) | 5), ((5 << 16) | 5), ((7 << 16) | 6), ((9 << 16) | 7)},   /*DDR3_1600J*/
 756         {((5 << 16) | 5), ((6 << 16) | 5), ((7 << 16) | 6), ((10 << 16) | 7)},  /*DDR3_1600K*/
 757
 758         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((8 << 16) | 7)},   /*DDR3_1866J*/
 759         {((5 << 16) | 5), ((5 << 16) | 5), ((7 << 16) | 6), ((8 << 16) | 7)},   /*DDR3_1866K*/
 760         {((6 << 16) | 5), ((6 << 16) | 5), ((7 << 16) | 6), ((9 << 16) | 7)},   /*DDR3_1866L*/
 761         {((6 << 16) | 5), ((6 << 16) | 5), ((8 << 16) | 6), ((10 << 16) | 7)},  /*DDR3_1866M*/
 762
 763         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((7 << 16) | 7)},   /*DDR3_2133K*/
 764         {((5 << 16) | 5), ((5 << 16) | 5), ((6 << 16) | 6), ((8 << 16) | 7)},   /*DDR3_2133L*/
 765         {((5 << 16) | 5), ((5 << 16) | 5), ((7 << 16) | 6), ((9 << 16) | 7)},   /*DDR3_2133M*/
 766         {((6 << 16) | 5), ((6 << 16) | 5), ((7 << 16) | 6), ((9 << 16) | 7)},   /*DDR3_2133N*/
 767
 768         {((6 << 16) | 5), ((6 << 16) | 5), ((8 << 16) | 6), ((10 << 16) | 7)}   /*DDR3_DEFAULT*/
 769 };
 770
 771 static const uint32_t ddr3_tRC_tFAW[22] = {
 772 /**    tRC    tFAW   */
 773         ((50 << 16) | 50),      /*DDR3_800D*/
 774         ((53 << 16) | 50),      /*DDR3_800E*/
 775
 776         ((49 << 16) | 50),      /*DDR3_1066E*/
 777         ((51 << 16) | 50),      /*DDR3_1066F*/
 778         ((53 << 16) | 50),      /*DDR3_1066G*/
 779
 780         ((47 << 16) | 45),      /*DDR3_1333F*/
 781         ((48 << 16) | 45),      /*DDR3_1333G*/
 782         ((50 << 16) | 45),      /*DDR3_1333H*/
 783         ((51 << 16) | 45),      /*DDR3_1333J*/
 784
 785         ((45 << 16) | 40),      /*DDR3_1600G*/
 786         ((47 << 16) | 40),      /*DDR3_1600H*/
 787         ((48 << 16) | 40),      /*DDR3_1600J*/
 788         ((49 << 16) | 40),      /*DDR3_1600K*/
 789
 790         ((45 << 16) | 35),      /*DDR3_1866J*/
 791         ((46 << 16) | 35),      /*DDR3_1866K*/
 792         ((47 << 16) | 35),      /*DDR3_1866L*/
 793         ((48 << 16) | 35),      /*DDR3_1866M*/
 794
 795         ((44 << 16) | 35),      /*DDR3_2133K*/
 796         ((45 << 16) | 35),      /*DDR3_2133L*/
 797         ((46 << 16) | 35),      /*DDR3_2133M*/
 798         ((47 << 16) | 35),      /*DDR3_2133N*/
 799
 800         ((53 << 16) | 50)       /*DDR3_DEFAULT*/
 801 };
 802
 803 /****************************************************************************
 804 *Internal sram us delay function
 805 *Cpu highest frequency is 1.6 GHz
 806 *1 cycle = 1/1.6 ns
 807 *1 us = 1000 ns = 1000 * 1.6 cycles = 1600 cycles
 808 ******************************************************************************/
 809 volatile uint32 DEFINE_PIE_DATA(loops_per_us);
 810 #define LPJ_100MHZ  999456UL
 811
 812 /*----------------------------------------------------------------------
 813 *Name   : void __sramlocalfunc ddr_delayus(uint32_t us)
 814 *Desc   : ddr ÑÓÊ±º¯Êý
 815 *Params  : uint32_t us  --ÑÓÊ±Ê±¼ä
 816 *Return  : void
 817 *Notes   : loops_per_us ÎªÈ«¾Ö±äÁ¿ ÐèÒª¸ù¾Ýarm freq¶ø¶¨
 818 *----------------------------------------------------------------------*/
 819 static void __sramfunc ddr_delayus(uint32 us)
 820 {
 821         do {
 822                 volatile unsigned int i = (DATA(loops_per_us) * us);
 823                 if (i < 7)
 824                         i = 7;
 825                 barrier();
 826                 asm volatile (".align 4; 1: subs %0, %0, #1; bne 1b;":"+r" (i));
 827         } while (0);
 828 }
 829
 830 /*----------------------------------------------------------------------
 831 *Name   : __sramfunc void ddr_copy(uint32 *pDest, uint32 *pSrc, uint32 words)
 832 *Desc   : ddr ¿½±´¼Ä´æÆ÷º¯Êý
 833 *Params  : pDest ->Ä¿±ê¼Ä´æÆ÷Ê×µØÖ·
 834 *          pSrc  ->Ô´±ê¼Ä´æÆ÷Ê×µØÖ·
 835 *          words ->¿½±´³¤¶È
 836 *Return  : void
 837 *Notes   :
 838 *----------------------------------------------------------------------*/
 839
 840 static __sramfunc void ddr_copy(uint32 *pDest, uint32 *pSrc, uint32 words)
 841 {
 842         uint32 i;
 843
 844         for (i = 0; i < words; i++) {
 845                 pDest[i] = pSrc[i];
 846         }
 847 }
 848
 849 /*----------------------------------------------------------------------
 850 *Name   : __sramfunc void ddr_move_to_Lowpower_state(void)
 851 *Desc   : pctl ½øÈë lowpower state
 852 *Params  : void
 853 *Return  : void
 854 *Notes   :
 855 *----------------------------------------------------------------------*/
 856 static __sramfunc void ddr_move_to_Lowpower_state(void)
 857 {
 858         volatile uint32 value;
 859
 860         while (1) {
 861                 value = pDDR_Reg->STAT.b.ctl_stat;
 862                 if (value == Low_power) {
 863                         break;
 864                 }
 865                 switch (value) {
 866                 case Init_mem:
 867                         pDDR_Reg->SCTL = CFG_STATE;
 868                         dsb();
 869                         while ((pDDR_Reg->STAT.b.ctl_stat) != Config)
 870                         ;
 871                 case Config:
 872                         pDDR_Reg->SCTL = GO_STATE;
 873                         dsb();
 874                         while ((pDDR_Reg->STAT.b.ctl_stat) != Access)
 875                         ;
 876                 case Access:
 877                         pDDR_Reg->SCTL = SLEEP_STATE;
 878                         dsb();
 879                         while ((pDDR_Reg->STAT.b.ctl_stat) != Low_power)
 880                         ;
 881                         break;
 882                 default:        /*Transitional state*/
 883                         break;
 884                 }
 885         }
 886 }
 887
 888 /*----------------------------------------------------------------------
 889 *Name   : __sramfunc void ddr_move_to_Access_state(void)
 890 *Desc   : pctl ½øÈë Access state
 891 *Params  : void
 892 *Return  : void
 893 *Notes   :
 894 *----------------------------------------------------------------------*/
 895 static __sramfunc void ddr_move_to_Access_state(void)
 896 {
 897         volatile uint32 value;
 898
 899         /*set auto self-refresh idle*/
 900         pDDR_Reg->MCFG1 =
 901             (pDDR_Reg->MCFG1 & 0xffffff00) | DATA(ddr_sr_idle) | (1 << 31);
 902         pDDR_Reg->MCFG = (pDDR_Reg->MCFG & 0xffff00ff) | (PD_IDLE << 8);
 903         while (1) {
 904                 value = pDDR_Reg->STAT.b.ctl_stat;
 905                 if ((value == Access)
 906                     || ((pDDR_Reg->STAT.b.lp_trig == 1)
 907                         && ((pDDR_Reg->STAT.b.ctl_stat) == Low_power))) {
 908                         break;
 909                 }
 910                 switch (value) {
 911                 case Low_power:
 912                         pDDR_Reg->SCTL = WAKEUP_STATE;
 913                         dsb();
 914                         while ((pDDR_Reg->STAT.b.ctl_stat) != Access)
 915                         ;
 916                         break;
 917                 case Init_mem:
 918                         pDDR_Reg->SCTL = CFG_STATE;
 919                         dsb();
 920                         while ((pDDR_Reg->STAT.b.ctl_stat) != Config)
 921                         ;
 922                 case Config:
 923                         pDDR_Reg->SCTL = GO_STATE;
 924                         dsb();
 925                         while (!(((pDDR_Reg->STAT.b.ctl_stat) == Access)
 926                                  || ((pDDR_Reg->STAT.b.lp_trig == 1)
 927                                      && ((pDDR_Reg->STAT.b.ctl_stat) ==
 928                                          Low_power))))
 929                     ;
 930                         break;
 931                 default:        /*Transitional state*/
 932                         break;
 933                 }
 934         }
 935         pGRF_Reg->GRF_SOC_CON[2] = (1 << 16 | 0);       /*de_hw_wakeup :enable auto sr if sr_idle != 0*/
 936 }
 937
 938 /*----------------------------------------------------------------------
 939 *Name   : __sramfunc void ddr_move_to_Config_state(void)
 940 *Desc   : pctl ½øÈë config state
 941 *Params  : void
 942 *Return  : void
 943 *Notes   :
 944 *----------------------------------------------------------------------*/
 945 static __sramfunc void ddr_move_to_Config_state(void)
 946 {
 947         volatile uint32 value;
 948         pGRF_Reg->GRF_SOC_CON[2] = (1 << 16 | 1);       /*hw_wakeup :disable auto sr*/
 949         while (1) {
 950                 value = pDDR_Reg->STAT.b.ctl_stat;
 951                 if (value == Config) {
 952                         break;
 953                 }
 954                 switch (value) {
 955                 case Low_power:
 956                         pDDR_Reg->SCTL = WAKEUP_STATE;
 957                         dsb();
 958                 case Access:
 959                 case Init_mem:
 960                         pDDR_Reg->SCTL = CFG_STATE;
 961                         dsb();
 962                         break;
 963                 default:        /*Transitional state*/
 964                         break;
 965                 }
 966         }
 967 }
 968
 969 /*----------------------------------------------------------------------
 970 *Name   : void __sramlocalfunc ddr_send_command(uint32 rank, uint32 cmd, uint32 arg)
 971 *Desc   : Í¨¹ýÐ´ pctl MCMD¼Ä´æÆ÷Ïòddr·¢ËÍÃüÁî
 972 *Params  : rank ->ddr rank Êý
 973 *          cmd  ->·¢ËÍÃüÁîÀàÐÍ
 974 *          arg  ->·¢ËÍµÄÊý¾Ý
 975 *Return  : void
 976 *Notes   : arg°üÀ¨bank_addrºÍcmd_addr
 977 *----------------------------------------------------------------------*/
 978 static void __sramfunc ddr_send_command(uint32 rank, uint32 cmd, uint32 arg)
 979 {
 980         pDDR_Reg->MCMD = (start_cmd | (rank << 20) | arg | cmd);
 981         dsb();
 982         while (pDDR_Reg->MCMD & start_cmd)
 983         ;
 984 }
 985
 986 __sramdata uint32 copy_data[8] = {
 987      0xffffffff, 0x00000000, 0x55555555, 0xAAAAAAAA,
 988         0xEEEEEEEE, 0x11111111, 0x22222222, 0xDDDDDDDD
 989 };
 990
 991  EXPORT_PIE_SYMBOL(copy_data[8]);
 992 static uint32 *p_copy_data;
 993
 994 /*----------------------------------------------------------------------
 995 Name    : uint32_t __sramlocalfunc ddr_data_training(void)
 996 Desc    : ¶Ôddr×ödata training
 997 Params  : void
 998 Return  : void
 999 Notes   : Ã»ÓÐ×ödata trainingÐ£Ñé
1000 ----------------------------------------------------------------------*/
1001 static uint32_t __sramfunc ddr_data_training(void)
1002 {
1003         uint32 value, dram_bw;
1004         value = pDDR_Reg->TREFI;
1005         pDDR_Reg->TREFI = 0;
1006         dram_bw = (pPHY_Reg->PHY_REG0 >> 4) & 0xf;
1007         pPHY_Reg->PHY_REG2 |= PHY_AUTO_CALIBRATION;
1008         dsb();
1009         /*wait echo byte DTDONE*/
1010         ddr_delayus(1);
1011         /*stop DTT*/
1012         while ((pPHY_Reg->PHY_REGff & 0xf) != dram_bw)
1013         ;
1014         pPHY_Reg->PHY_REG2 = (pPHY_Reg->PHY_REG2 & (~0x1));
1015         /*send some auto refresh to complement the lost while DTT*/
1016         ddr_send_command(3, PREA_cmd, 0);
1017         ddr_send_command(3, REF_cmd, 0);
1018         ddr_send_command(3, REF_cmd, 0);
1019
1020         /*resume auto refresh*/
1021         pDDR_Reg->TREFI = value;
1022         return 0;
1023 }
1024
1025 /*----------------------------------------------------------------------
1026 Name    : void __sramlocalfunc ddr_set_dll_bypass(uint32 freq)
1027 Desc    : ÉèÖÃPHY dll ¹¤×÷Ä£Ê½
1028 Params  : freq -> ddr¹¤×÷ÆµÂÊ
1029 Return  : void
1030 Notes   :
1031 ----------------------------------------------------------------------*/
1032 static void __sramfunc ddr_set_dll_bypass(uint32 freq)
1033 {
1034 #if defined (PHY_RX_PHASE_CAL)
1035         uint32 phase_90, dll_set, de_skew;
1036
1037         phase_90 = 1000000 / freq / 4;
1038         dll_set = (phase_90 - 300 + (0x7*PHY_DE_SKEW_STEP)) / (phase_90 / 4);
1039         de_skew = (phase_90 - 300 + (0x7*PHY_DE_SKEW_STEP) - ((phase_90 / 4) * dll_set));
1040         if (de_skew > PHY_DE_SKEW_STEP * 15) {
1041                 if (dll_set == 3) {
1042                         de_skew = 15;
1043                 } else {
1044                         dll_set += 1;
1045                         de_skew = 0;
1046                 }
1047         } else {
1048                 de_skew = de_skew / PHY_DE_SKEW_STEP;
1049         }
1050
1051         pPHY_Reg->PHY_REG28 = dll_set;/*rx dll 45¡ãdelay*/
1052         pPHY_Reg->PHY_REG38 = dll_set;/*rx dll 45¡ãdelay*/
1053         pPHY_Reg->PHY_REG48 = dll_set;/*rx dll 45¡ãdelay*/
1054         pPHY_Reg->PHY_REG58 = dll_set;/*rx dll 45¡ãdelay*/
1055         pPHY_Reg->PHY_REG_skew[(0x324-0x2c0)/4] = 0x7 | (de_skew << 4);
1056         pPHY_Reg->PHY_REG_skew[(0x350-0x2c0)/4] = 0x7 | (de_skew << 4);
1057         pPHY_Reg->PHY_REG_skew[(0x37c-0x2c0)/4] = 0x7 | (de_skew << 4);
1058         pPHY_Reg->PHY_REG_skew[(0x3a8-0x2c0)/4] = 0x7 | (de_skew << 4);
1059 #else
1060         uint32 phase;
1061         if (freq < 350) {
1062                 phase = 3;
1063         } else if (freq < 600) {
1064                 phase = 2;
1065         } else
1066                 phase = 1;
1067         pPHY_Reg->PHY_REG28 = phase;    /*rx dll 45¡ãdelay*/
1068         pPHY_Reg->PHY_REG38 = phase;    /*rx dll 45¡ãdelay*/
1069         pPHY_Reg->PHY_REG48 = phase;    /*rx dll 45¡ãdelay*/
1070         pPHY_Reg->PHY_REG58 = phase;    /*rx dll 45¡ãdelay*/
1071 #endif
1072         if (freq <= PHY_DLL_DISABLE_FREQ) {
1073                 pPHY_Reg->PHY_REGDLL |= 0x1F;   /*TX DLL bypass */
1074         } else {
1075                 pPHY_Reg->PHY_REGDLL &= ~0x1F;  /* TX DLL bypass*/
1076         }
1077
1078         dsb();
1079 }
1080
1081 static noinline uint32 ddr_get_pll_freq(PLL_ID pll_id)  /*APLL-1;CPLL-2;DPLL-3;GPLL-4*/
1082 {
1083         uint32 ret = 0;
1084
1085         if (((pCRU_Reg->CRU_MODE_CON >> (pll_id * 4)) & 1) == 1)        /* DPLL Normal mode*/
1086                 ret = 24 * ((pCRU_Reg->CRU_PLL_CON[pll_id][0] & 0xfff)) /* NF = 2*(CLKF+1)*/
1087                     / ((pCRU_Reg->CRU_PLL_CON[pll_id][1] & 0x3f)
1088                        * ((pCRU_Reg->CRU_PLL_CON[pll_id][0] >> 12) & 0x7) * ((pCRU_Reg->CRU_PLL_CON[pll_id][1] >> 6) & 0x7));   /* OD = 2^CLKOD*/
1089         else
1090                 ret = 24;
1091
1092         return ret;
1093 }
1094
1095 static __sramdata uint32 clkFbDiv;
1096 static __sramdata uint32 clkPostDiv1;
1097 static __sramdata uint32 clkPostDiv2;
1098
1099 /*****************************************
1100 *REFDIV   FBDIV     POSTDIV1/POSTDIV2      FOUTPOSTDIV           freq Step        FOUTPOSRDIV            finally use
1101 *==================================================================================================================
1102 *1        17 - 66   4                      100MHz - 400MHz          6MHz          200MHz  <= 300MHz             <= 150MHz
1103 *1        17 - 66   3                      133MHz - 533MHz          8MHz
1104 *1        17 - 66   2                      200MHz - 800MHz          12MHz         300MHz  <= 600MHz      150MHz <= 300MHz
1105 *1        17 - 66   1                      400MHz - 1600MHz         24MHz         600MHz  <= 1200MHz     300MHz <= 600MHz
1106 *******************************************/
1107 /*for minimum jitter operation, the highest VCO and FREF frequencies should be used.*/
1108 /*----------------------------------------------------------------------
1109 *Name    : uint32_t __sramlocalfunc ddr_set_pll(uint32_t nMHz, uint32_t set)
1110 *Desc    : ÉèÖÃddr pll
1111 *Params  : nMHZ -> ddr¹¤×÷ÆµÂÊ
1112 *          set  ->0»ñÈ¡ÉèÖÃµÄÆµÂÊÐÅÏ¢
1113 *                 1ÉèÖÃddr pll
1114 *Return  : ÉèÖÃµÄÆµÂÊÖµ
1115 *Notes   : ÔÚ±äÆµÊ±ÐèÒªÏÈset=0µ÷ÓÃÒ»´Îddr_set_pll£¬ÔÙset=1 µ÷ÓÃddr_set_pll
1116 -*---------------------------------------------------------------------*/
1117 static uint32 __sramfunc ddr_set_pll(uint32 nMHz, uint32 set)
1118 {
1119         uint32 ret = 0;
1120         int delay = 1000;
1121         uint32 pll_id = 1;      /*DPLL*/
1122
1123         if (nMHz == 24) {
1124                 ret = 24;
1125                 goto out;
1126         }
1127         if (!set) {
1128                 if (nMHz <= 150) {      /*Êµ¼ÊÊä³öÆµÂÊ<300*/
1129                         clkPostDiv1 = 6;
1130                 } else if (nMHz <= 200) {
1131                         clkPostDiv1 = 4;
1132                 } else if (nMHz <= 300) {
1133                         clkPostDiv1 = 3;
1134                 } else if (nMHz <= 450) {
1135                         clkPostDiv1 = 2;
1136                 } else {
1137                         clkPostDiv1 = 1;
1138                 }
1139                 clkPostDiv2 = 1;
1140                 clkFbDiv = (nMHz * 2 * DDR_PLL_REFDIV * clkPostDiv1 * clkPostDiv2) / 24;        /*×îºóËÍÈëddrµÄÊÇÔÙ¾¹ý2·ÖÆµ*/
1141                 ret =
1142                     (24 * clkFbDiv) / (2 * DDR_PLL_REFDIV * clkPostDiv1 *
1143                                        clkPostDiv2);
1144         } else {
1145                 pCRU_Reg->CRU_MODE_CON = (0x1 << ((pll_id * 4) + 16)) | (0x0 << (pll_id * 4));  /*PLL slow-mode*/
1146
1147                 pCRU_Reg->CRU_PLL_CON[pll_id][0] =
1148                     FBDIV(clkFbDiv) | POSTDIV1(clkPostDiv1);
1149                 pCRU_Reg->CRU_PLL_CON[pll_id][1] = REFDIV(DDR_PLL_REFDIV) | POSTDIV2(clkPostDiv2) | (0x10001 << 12);    /*interger mode*/
1150
1151                 ddr_delayus(1);
1152
1153                 while (delay > 0) {
1154                         ddr_delayus(1);
1155                         if (pCRU_Reg->CRU_PLL_CON[pll_id][1] & (PLL_LOCK_STATUS))       /*wait for pll locked*/
1156                                 break;
1157                         delay--;
1158                 }
1159
1160                 pCRU_Reg->CRU_CLKSEL_CON[26] = ((0x3 << 16) | 0x0);     /*clk_ddr_src:clk_ddrphy = 1:1*/
1161                 pCRU_Reg->CRU_MODE_CON = (0x1 << ((pll_id * 4) + 16)) | (0x1 << (pll_id * 4));  /*PLL normal*/
1162         }
1163 out:
1164         return ret;
1165 }
1166
1167 uint32 PIE_FUNC(ddr_set_pll)(uint32 nMHz, uint32 set)
1168 {
1169         return ddr_set_pll(nMHz, set);
1170 }
1171
1172 EXPORT_PIE_SYMBOL(FUNC(ddr_set_pll));
1173 static uint32(*p_ddr_set_pll) (uint32 nMHz, uint32 set);
1174
1175 /*----------------------------------------------------------------------
1176 *Name    : uint32_t ddr_get_parameter(uint32_t nMHz)
1177 *Desc    : »ñÈ¡ÅäÖÃ²ÎÊý
1178 *Params  : nMHZ -> ddr¹¤×÷ÆµÂÊ
1179 *Return  : 0 ³É¹¦
1180 *          -1 Ê§°Ü
1181 *          -4 ÆµÂÊÖµ³¬¹ý¿ÅÁ£×î´óÆµÂÊ
1182 *Notes   :
1183 *----------------------------------------------------------------------*/
1184 static uint32 ddr_get_parameter(uint32 nMHz)
1185 {
1186         uint32_t tmp;
1187         uint32_t ret = 0;
1188         uint32_t al;
1189         uint32_t bl;
1190         uint32_t cl;
1191         uint32_t cwl;
1192         uint32_t bl_tmp;
1193         PCTL_TIMING_T *p_pctl_timing = &(p_ddr_reg->pctl_timing);
1194         NOC_TIMING_T *p_noc_timing = &(p_ddr_reg->noc_timing);
1195
1196         p_pctl_timing->togcnt1u = nMHz;
1197         p_pctl_timing->togcnt100n = nMHz / 10;
1198         p_pctl_timing->tinit = 200;
1199
1200         if (p_ddr_reg->mem_type == DDR3) {
1201                 if (p_ddr_reg->ddr_speed_bin > DDR3_DEFAULT) {
1202                         ret = -1;
1203                         goto out;
1204                 }
1205 #define DDR3_tREFI_7_8_us    (78)
1206 #define DDR3_tMRD            (4)
1207 #define DDR3_tRFC_512Mb      (90)
1208 #define DDR3_tRFC_1Gb        (110)
1209 #define DDR3_tRFC_2Gb        (160)
1210 #define DDR3_tRFC_4Gb        (300)
1211 #define DDR3_tRFC_8Gb        (350)
1212 #define DDR3_tRTW            (2)        /*register min valid value*/
1213 #define DDR3_tRAS            (37)
1214 #define DDR3_tRRD            (10)
1215 #define DDR3_tRTP            (7)
1216 #define DDR3_tWR             (15)
1217 #define DDR3_tWTR            (7)
1218 #define DDR3_tXP             (7)
1219 #define DDR3_tXPDLL          (24)
1220 #define DDR3_tZQCS           (80)
1221 #define DDR3_tZQCSI          (10000)
1222 #define DDR3_tDQS            (1)
1223 #define DDR3_tCKSRE          (10)
1224 #define DDR3_tCKE_400MHz     (7)
1225 #define DDR3_tCKE_533MHz     (6)
1226 #define DDR3_tMOD            (15)
1227 #define DDR3_tRSTL           (100)
1228 #define DDR3_tZQCL           (320)
1229 #define DDR3_tDLLK           (512)
1230                 p_pctl_timing->trsth = 500;
1231                 al = 0;
1232                 bl = 8;
1233                 if (nMHz <= 330) {
1234                         tmp = 0;
1235                 } else if (nMHz <= 400) {
1236                         tmp = 1;
1237                 } else if (nMHz <= 533) {
1238                         tmp = 2;
1239                 } else {        /*666MHz*/
1240                         tmp = 3;
1241                 }
1242                 if (nMHz <= DDR3_DDR2_DLL_DISABLE_FREQ) {       /*when dll bypss cl = cwl = 6*/
1243                         cl = 6;
1244                         cwl = 6;
1245                 } else {
1246                         cl = ddr3_cl_cwl[p_ddr_reg->ddr_speed_bin][tmp] >> 16;
1247                         cwl =
1248                             ddr3_cl_cwl[p_ddr_reg->ddr_speed_bin][tmp] & 0x0ff;
1249                 }
1250                 if (cl == 0) {
1251                         ret = -4;       /*³¬¹ý¿ÅÁ£µÄ×î´óÆµÂÊ*/
1252                 }
1253                 if (nMHz <= DDR3_DDR2_ODT_DISABLE_FREQ) {
1254                         p_ddr_reg->ddrMR[1] = DDR3_DS_40 | DDR3_Rtt_Nom_DIS;
1255                 } else {
1256                         p_ddr_reg->ddrMR[1] = DDR3_DS_40 | DDR3_Rtt_Nom_120;
1257                 }
1258                 p_ddr_reg->ddrMR[2] = DDR3_MR2_CWL(cwl) /* | DDR3_Rtt_WR_60 */ ;
1259                 p_ddr_reg->ddrMR[3] = 0;
1260                 /**************************************************
1261                 * PCTL Timing
1262                 **************************************************/
1263                 /*
1264                  * tREFI, average periodic refresh interval, 7.8us
1265                  */
1266                 p_pctl_timing->trefi = DDR3_tREFI_7_8_us;
1267                 /*
1268                  * tMRD, 4 tCK
1269                  */
1270                 p_pctl_timing->tmrd = DDR3_tMRD & 0x7;
1271                 /*
1272                  * tRFC, 90ns(512Mb),110ns(1Gb),160ns(2Gb),300ns(4Gb),350ns(8Gb)
1273                  */
1274                 if (p_ddr_reg->ddr_capability_per_die <= 0x4000000) {   /*512Mb 90ns*/
1275                         tmp = DDR3_tRFC_512Mb;
1276                 } else if (p_ddr_reg->ddr_capability_per_die <= 0x8000000) {    /*1Gb 110ns*/
1277                         tmp = DDR3_tRFC_1Gb;
1278                 } else if (p_ddr_reg->ddr_capability_per_die <= 0x10000000) {   /*2Gb 160ns*/
1279                         tmp = DDR3_tRFC_2Gb;
1280                 } else if (p_ddr_reg->ddr_capability_per_die <= 0x20000000) {/*4Gb 300ns*/
1281                         tmp = DDR3_tRFC_4Gb;
1282                 } else{         /*8Gb  350ns*/
1283                         tmp = DDR3_tRFC_8Gb;
1284                 }
1285                 p_pctl_timing->trfc = (tmp * nMHz + 999) / 1000;
1286                 /*
1287                  * tXSR, =tDLLK=512 tCK
1288                  */
1289                 p_pctl_timing->texsr = DDR3_tDLLK;
1290                 /*
1291                  * tRP=CL
1292                  */
1293                 p_pctl_timing->trp = cl;
1294                 /*
1295                  * WrToMiss=WL*tCK + tWR + tRP + tRCD
1296                  */
1297                 p_noc_timing->b.WrToMiss =
1298                     ((cwl + ((DDR3_tWR * nMHz + 999) / 1000) + cl + cl) & 0x3F);
1299                 /*
1300                  * tRC=tRAS+tRP
1301                  */
1302                 p_pctl_timing->trc =
1303                     ((((ddr3_tRC_tFAW[p_ddr_reg->ddr_speed_bin] >> 16) * nMHz +
1304                        999) / 1000) & 0x3F);
1305                 p_noc_timing->b.ActToAct =
1306                     ((((ddr3_tRC_tFAW[p_ddr_reg->ddr_speed_bin] >> 16) * nMHz +
1307                        999) / 1000) & 0x3F);
1308
1309                 p_pctl_timing->trtw = (cl + 2 - cwl);   /*DDR3_tRTW*/
1310                 p_noc_timing->b.RdToWr = ((cl + 2 - cwl) & 0x1F);
1311                 p_pctl_timing->tal = al;
1312                 p_pctl_timing->tcl = cl;
1313                 p_pctl_timing->tcwl = cwl;
1314                 /*
1315                  * tRAS, 37.5ns(400MHz)     37.5ns(533MHz)
1316                  */
1317                 p_pctl_timing->tras =
1318                     (((DDR3_tRAS * nMHz + (nMHz >> 1) + 999) / 1000) & 0x3F);
1319                 /*
1320                  * tRCD=CL
1321                  */
1322                 p_pctl_timing->trcd = cl;
1323                 /*
1324                  * tRRD = max(4nCK, 7.5ns), DDR3-1066(1K), DDR3-1333(2K), DDR3-1600(2K)
1325                  *        max(4nCK, 10ns), DDR3-800(1K,2K), DDR3-1066(2K)
1326                  *        max(4nCK, 6ns), DDR3-1333(1K), DDR3-1600(1K)
1327                  *
1328                  */
1329                 tmp = ((DDR3_tRRD * nMHz + 999) / 1000);
1330                 if (tmp < 4) {
1331                         tmp = 4;
1332                 }
1333                 p_pctl_timing->trrd = (tmp & 0xF);
1334                 /*
1335                  * tRTP, max(4 tCK,7.5ns)
1336                  */
1337                 tmp = ((DDR3_tRTP * nMHz + (nMHz >> 1) + 999) / 1000);
1338                 if (tmp < 4) {
1339                         tmp = 4;
1340                 }
1341                 p_pctl_timing->trtp = tmp & 0xF;
1342                 /*
1343                  * RdToMiss=tRTP+tRP + tRCD - (BL/2 * tCK)
1344                  */
1345                 p_noc_timing->b.RdToMiss = ((tmp + cl + cl - (bl >> 1)) & 0x3F);
1346                 /*
1347                  * tWR, 15ns
1348                  */
1349                 tmp = ((DDR3_tWR * nMHz + 999) / 1000);
1350                 p_pctl_timing->twr = tmp & 0x1F;
1351                 if (tmp < 9)
1352                         tmp = tmp - 4;
1353                 else
1354                         tmp = tmp >> 1;
1355                 p_ddr_reg->ddrMR[0] = DDR3_BL8 | DDR3_CL(cl) | DDR3_WR(tmp);
1356
1357                 /*
1358                  * tWTR, max(4 tCK,7.5ns)
1359                  */
1360                 tmp = ((DDR3_tWTR * nMHz + (nMHz >> 1) + 999) / 1000);
1361                 if (tmp < 4) {
1362                         tmp = 4;
1363                 }
1364                 p_pctl_timing->twtr = tmp & 0xF;
1365                 p_noc_timing->b.WrToRd = ((tmp + cwl) & 0x1F);
1366                 /*
1367                  * tXP, max(3 tCK, 7.5ns)(<933MHz)
1368                  */
1369                 tmp = ((DDR3_tXP * nMHz + (nMHz >> 1) + 999) / 1000);
1370                 if (tmp < 3) {
1371                         tmp = 3;
1372                 }
1373                 p_pctl_timing->txp = tmp & 0x7;
1374                 /*
1375                  * tXPDLL, max(10 tCK,24ns)
1376                  */
1377                 tmp = ((DDR3_tXPDLL * nMHz + 999) / 1000);
1378                 if (tmp < 10) {
1379                         tmp = 10;
1380                 }
1381                 p_pctl_timing->txpdll = tmp & 0x3F;
1382                 /*
1383                  * tZQCS, max(64 tCK, 80ns)
1384                  */
1385                 tmp = ((DDR3_tZQCS * nMHz + 999) / 1000);
1386                 if (tmp < 64) {
1387                         tmp = 64;
1388                 }
1389                 p_pctl_timing->tzqcs = tmp & 0x7F;
1390                 /*
1391                  * tZQCSI,
1392                  */
1393                 p_pctl_timing->tzqcsi = DDR3_tZQCSI;
1394                 /*
1395                  * tDQS,
1396                  */
1397                 p_pctl_timing->tdqs = DDR3_tDQS;
1398                 /*
1399                  * tCKSRE, max(5 tCK, 10ns)
1400                  */
1401                 tmp = ((DDR3_tCKSRE * nMHz + 999) / 1000);
1402                 if (tmp < 5) {
1403                         tmp = 5;
1404                 }
1405                 p_pctl_timing->tcksre = tmp & 0x1F;
1406                 /*
1407                  * tCKSRX, max(5 tCK, 10ns)
1408                  */
1409                 p_pctl_timing->tcksrx = tmp & 0x1F;
1410                 /*
1411                  * tCKE, max(3 tCK,7.5ns)(400MHz) max(3 tCK,5.625ns)(533MHz)
1412                  */
1413                 if (nMHz >= 533) {
1414                         tmp = ((DDR3_tCKE_533MHz * nMHz + 999) / 1000);
1415                 } else {
1416                         tmp =
1417                             ((DDR3_tCKE_400MHz * nMHz + (nMHz >> 1) +
1418                               999) / 1000);
1419                 }
1420                 if (tmp < 3) {
1421                         tmp = 3;
1422                 }
1423                 p_pctl_timing->tcke = tmp & 0x7;
1424                 /*
1425                  * tCKESR, =tCKE + 1tCK
1426                  */
1427                 p_pctl_timing->tckesr = (tmp + 1) & 0xF;
1428                 /*
1429                  * tMOD, max(12 tCK,15ns)
1430                  */
1431                 tmp = ((DDR3_tMOD * nMHz + 999) / 1000);
1432                 if (tmp < 12) {
1433                         tmp = 12;
1434                 }
1435                 p_pctl_timing->tmod = tmp & 0x1F;
1436                 /*
1437                  * tRSTL, 100ns
1438                  */
1439                 p_pctl_timing->trstl =
1440                     ((DDR3_tRSTL * nMHz + 999) / 1000) & 0x7F;
1441                 /*
1442                  * tZQCL, max(256 tCK, 320ns)
1443                  */
1444                 tmp = ((DDR3_tZQCL * nMHz + 999) / 1000);
1445                 if (tmp < 256) {
1446                         tmp = 256;
1447                 }
1448                 p_pctl_timing->tzqcl = tmp & 0x3FF;
1449                 /*
1450                  * tMRR, 0 tCK
1451                  */
1452                 p_pctl_timing->tmrr = 0;
1453                 /*
1454                  * tDPD, 0
1455                  */
1456                 p_pctl_timing->tdpd = 0;
1457
1458                 /**************************************************
1459                 *NOC Timing
1460                 **************************************************/
1461                 p_noc_timing->b.BurstLen = ((bl >> 1) & 0x7);
1462         } else if (p_ddr_reg->mem_type == LPDDR2) {
1463 #define LPDDR2_tREFI_3_9_us    (38)     /*unit 100ns*/
1464 #define LPDDR2_tREFI_7_8_us    (78)     /*unit 100ns*/
1465 #define LPDDR2_tMRD            (5)      /*tCK*/
1466 #define LPDDR2_tRFC_8Gb        (210)    /*ns*/
1467 #define LPDDR2_tRFC_4Gb        (130)    /*ns*/
1468 #define LPDDR2_tRP_4_BANK               (24)    /*ns*/
1469 #define LPDDR2_tRPab_SUB_tRPpb_4_BANK   (0)
1470 #define LPDDR2_tRP_8_BANK               (27)    /*ns*/
1471 #define LPDDR2_tRPab_SUB_tRPpb_8_BANK   (3)
1472 #define LPDDR2_tRTW          (1)        /*tCK register min valid value*/
1473 #define LPDDR2_tRAS          (42)       /*ns*/
1474 #define LPDDR2_tRCD          (24)       /*ns*/
1475 #define LPDDR2_tRRD          (10)       /*ns*/
1476 #define LPDDR2_tRTP          (7)        /*ns*/
1477 #define LPDDR2_tWR           (15)       /*ns*/
1478 #define LPDDR2_tWTR_GREAT_200MHz         (7)    /*ns*/
1479 #define LPDDR2_tWTR_LITTLE_200MHz        (10)   /*ns*/
1480 #define LPDDR2_tXP           (7)        /*ns*/
1481 #define LPDDR2_tXPDLL        (0)
1482 #define LPDDR2_tZQCS         (90)       /*ns*/
1483 #define LPDDR2_tZQCSI        (0)
1484 #define LPDDR2_tDQS          (1)
1485 #define LPDDR2_tCKSRE        (1)        /*tCK*/
1486 #define LPDDR2_tCKSRX        (2)        /*tCK*/
1487 #define LPDDR2_tCKE          (3)        /*tCK*/
1488 #define LPDDR2_tMOD          (0)
1489 #define LPDDR2_tRSTL         (0)
1490 #define LPDDR2_tZQCL         (360)      /*ns*/
1491 #define LPDDR2_tMRR          (2)        /*tCK*/
1492 #define LPDDR2_tCKESR        (15)       /*ns*/
1493 #define LPDDR2_tDPD_US       (500)
1494 #define LPDDR2_tFAW_GREAT_200MHz    (50)        /*ns*/
1495 #define LPDDR2_tFAW_LITTLE_200MHz   (60)        /*ns*/
1496 #define LPDDR2_tDLLK         (2)        /*tCK*/
1497 #define LPDDR2_tDQSCK_MAX    (3)        /*tCK*/
1498 #define LPDDR2_tDQSCK_MIN    (0)        /*tCK*/
1499 #define LPDDR2_tDQSS         (1)        /*tCK*/
1500
1501                 uint32 trp_tmp;
1502                 uint32 trcd_tmp;
1503                 uint32 tras_tmp;
1504                 uint32 trtp_tmp;
1505                 uint32 twr_tmp;
1506
1507                 al = 0;
1508                 if (nMHz >= 200) {
1509                         bl = 4; /*you can change burst here*/
1510                 } else {
1511                         bl = 8; /* freq < 200MHz, BL fixed 8*/
1512                 }
1513                 /*     1066 933 800 667 533 400 333
1514                  * RL,   8   7   6   5   4   3   3
1515                  * WL,   4   4   3   2   2   1   1
1516                  */
1517                 if (nMHz <= 200) {
1518                         cl = 3;
1519                         cwl = 1;
1520                         p_ddr_reg->ddrMR[2] = LPDDR2_RL3_WL1;
1521                 } else if (nMHz <= 266) {
1522                         cl = 4;
1523                         cwl = 2;
1524                         p_ddr_reg->ddrMR[2] = LPDDR2_RL4_WL2;
1525                 } else if (nMHz <= 333) {
1526                         cl = 5;
1527                         cwl = 2;
1528                         p_ddr_reg->ddrMR[2] = LPDDR2_RL5_WL2;
1529                 } else if (nMHz <= 400) {
1530                         cl = 6;
1531                         cwl = 3;
1532                         p_ddr_reg->ddrMR[2] = LPDDR2_RL6_WL3;
1533                 } else if (nMHz <= 466) {
1534                         cl = 7;
1535                         cwl = 4;
1536                         p_ddr_reg->ddrMR[2] = LPDDR2_RL7_WL4;
1537                 } else {                /*(nMHz<=1066)*/
1538                         cl = 8;
1539                         cwl = 4;
1540                         p_ddr_reg->ddrMR[2] = LPDDR2_RL8_WL4;
1541                 }
1542                 p_ddr_reg->ddrMR[3] = LPDDR2_DS_34;
1543                 p_ddr_reg->ddrMR[0] = 0;
1544                 /**************************************************
1545                 * PCTL Timing
1546                 **************************************************/
1547                 /*
1548                  * tREFI, average periodic refresh interval, 15.6us(<256Mb) 7.8us(256Mb-1Gb) 3.9us(2Gb-8Gb)
1549                  */
1550                 if (p_ddr_reg->ddr_capability_per_die >= 0x10000000) {  /*2Gb*/
1551                         p_pctl_timing->trefi = LPDDR2_tREFI_3_9_us;
1552                 } else {
1553                         p_pctl_timing->trefi = LPDDR2_tREFI_7_8_us;
1554                 }
1555
1556                 /*
1557                  * tMRD, (=tMRW), 5 tCK
1558                  */
1559                 p_pctl_timing->tmrd = LPDDR2_tMRD & 0x7;
1560                 /*
1561                  * tRFC, 90ns(<=512Mb) 130ns(1Gb-4Gb) 210ns(8Gb)
1562                  */
1563                 if (p_ddr_reg->ddr_capability_per_die >= 0x40000000) {  /*8Gb*/
1564                         p_pctl_timing->trfc =
1565                             (LPDDR2_tRFC_8Gb * nMHz + 999) / 1000;
1566                         /*
1567                          * tXSR, max(2tCK,tRFC+10ns)
1568                          */
1569                         tmp = (((LPDDR2_tRFC_8Gb + 10) * nMHz + 999) / 1000);
1570                 } else {
1571                         p_pctl_timing->trfc =
1572                             (LPDDR2_tRFC_4Gb * nMHz + 999) / 1000;
1573                         tmp = (((LPDDR2_tRFC_4Gb + 10) * nMHz + 999) / 1000);
1574                 }
1575                 if (tmp < 2) {
1576                         tmp = 2;
1577                 }
1578                 p_pctl_timing->texsr = tmp & 0x3FF;
1579                 /*
1580                  * tRP, max(3tCK, 4-bank:15ns(Fast) 18ns(Typ) 24ns(Slow), 8-bank:18ns(Fast) 21ns(Typ) 27ns(Slow))
1581                  */
1582                 trp_tmp = ((LPDDR2_tRP_8_BANK * nMHz + 999) / 1000);
1583                 if (trp_tmp < 3) {
1584                         trp_tmp = 3;
1585                 }
1586                 p_pctl_timing->trp =
1587                     ((((LPDDR2_tRPab_SUB_tRPpb_8_BANK * nMHz +
1588                         999) / 1000) & 0x3) << 16) | (trp_tmp & 0xF);
1589
1590                 /*
1591                  * tRAS, max(3tCK,42ns)
1592                  */
1593                 tras_tmp = ((LPDDR2_tRAS * nMHz + 999) / 1000);
1594                 if (tras_tmp < 3) {
1595                         tras_tmp = 3;
1596                 }
1597                 p_pctl_timing->tras = (tras_tmp & 0x3F);
1598
1599                 /*
1600                  * tRCD, max(3tCK, 15ns(Fast) 18ns(Typ) 24ns(Slow))
1601                  */
1602                 trcd_tmp = ((LPDDR2_tRCD * nMHz + 999) / 1000);
1603                 if (trcd_tmp < 3) {
1604                         trcd_tmp = 3;
1605                 }
1606                 p_pctl_timing->trcd = (trcd_tmp & 0xF);
1607                 /*
1608                  * tRTP, max(2tCK, 7.5ns)
1609                  */
1610                 trtp_tmp = ((LPDDR2_tRTP * nMHz + (nMHz >> 1) + 999) / 1000);
1611                 if (trtp_tmp < 2) {
1612                         trtp_tmp = 2;
1613                 }
1614                 p_pctl_timing->trtp = trtp_tmp & 0xF;
1615                 /*
1616                  * tWR, max(3tCK,15ns)
1617                  */
1618                 twr_tmp = ((LPDDR2_tWR * nMHz + 999) / 1000);
1619                 if (twr_tmp < 3) {
1620                         twr_tmp = 3;
1621                 }
1622                 p_pctl_timing->twr = twr_tmp & 0x1F;
1623                 bl_tmp =
1624                     (bl ==
1625                      16) ? LPDDR2_BL16 : ((bl == 8) ? LPDDR2_BL8 : LPDDR2_BL4);
1626                 p_ddr_reg->ddrMR[1] = bl_tmp | LPDDR2_nWR(twr_tmp);
1627
1628                 /*
1629                  * WrToMiss=WL*tCK + tDQSS + tWR + tRP + tRCD
1630                  */
1631                 p_noc_timing->b.WrToMiss =
1632                     ((cwl + LPDDR2_tDQSS + twr_tmp + trp_tmp +
1633                       trcd_tmp) & 0x3F);
1634                 /*
1635                  * RdToMiss=tRTP + tRP + tRCD - (BL/2 * tCK)
1636                  */
1637                 p_noc_timing->b.RdToMiss =
1638                     ((trtp_tmp + trp_tmp + trcd_tmp - (bl >> 1)) & 0x3F);
1639                 /*
1640                  * tRC=tRAS+tRP
1641                  */
1642                 p_pctl_timing->trc = ((tras_tmp + trp_tmp) & 0x3F);
1643                 p_noc_timing->b.ActToAct = ((tras_tmp + trp_tmp) & 0x3F);
1644                 /*
1645                  * RdToWr=RL+tDQSCK-WL
1646                  */
1647                 p_pctl_timing->trtw = (cl + LPDDR2_tDQSCK_MAX + (bl / 2) + 1 - cwl);    /*LPDDR2_tRTW*/
1648                 p_noc_timing->b.RdToWr =
1649                     ((cl + LPDDR2_tDQSCK_MAX + 1 - cwl) & 0x1F);
1650                 p_pctl_timing->tal = al;
1651                 p_pctl_timing->tcl = cl;
1652                 p_pctl_timing->tcwl = cwl;
1653                 /*
1654                  * tRRD, max(2tCK,10ns)
1655                  */
1656                 tmp = ((LPDDR2_tRRD * nMHz + 999) / 1000);
1657                 if (tmp < 2) {
1658                         tmp = 2;
1659                 }
1660                 p_pctl_timing->trrd = (tmp & 0xF);
1661                 /*
1662                  * tWTR, max(2tCK, 7.5ns(533-266MHz)  10ns(200-166MHz))
1663                  */
1664                 if (nMHz > 200) {
1665                         tmp =
1666                             ((LPDDR2_tWTR_GREAT_200MHz * nMHz + (nMHz >> 1) +
1667                               999) / 1000);
1668                 } else {
1669                         tmp = ((LPDDR2_tWTR_LITTLE_200MHz * nMHz + 999) / 1000);
1670                 }
1671                 if (tmp < 2) {
1672                         tmp = 2;
1673                 }
1674                 p_pctl_timing->twtr = tmp & 0xF;
1675                 /*
1676                  * WrToRd=WL+tDQSS+tWTR
1677                  */
1678                 p_noc_timing->b.WrToRd = ((cwl + LPDDR2_tDQSS + tmp) & 0x1F);
1679                 /*
1680                  * tXP, max(2tCK,7.5ns)
1681                  */
1682                 tmp = ((LPDDR2_tXP * nMHz + (nMHz >> 1) + 999) / 1000);
1683                 if (tmp < 2) {
1684                         tmp = 2;
1685                 }
1686                 p_pctl_timing->txp = tmp & 0x7;
1687                 /*
1688                  * tXPDLL, 0ns
1689                  */
1690                 p_pctl_timing->txpdll = LPDDR2_tXPDLL;
1691                 /*
1692                  * tZQCS, 90ns
1693                  */
1694                 p_pctl_timing->tzqcs =
1695                     ((LPDDR2_tZQCS * nMHz + 999) / 1000) & 0x7F;
1696                 /*
1697                  * tZQCSI,
1698                  */
1699                 /*if (pDDR_Reg->MCFG &= lpddr2_s4) {*/
1700                 if (1) {
1701                         p_pctl_timing->tzqcsi = LPDDR2_tZQCSI;
1702                 } else {
1703                         p_pctl_timing->tzqcsi = 0;
1704                 }
1705                 /*
1706                  * tDQS,
1707                  */
1708                 p_pctl_timing->tdqs = LPDDR2_tDQS;
1709                 /*
1710                  * tCKSRE, 1 tCK
1711                  */
1712                 p_pctl_timing->tcksre = LPDDR2_tCKSRE;
1713                 /*
1714                  * tCKSRX, 2 tCK
1715                  */
1716                 p_pctl_timing->tcksrx = LPDDR2_tCKSRX;
1717                 /*
1718                  * tCKE, 3 tCK
1719                  */
1720                 p_pctl_timing->tcke = LPDDR2_tCKE;
1721                 /*
1722                  * tMOD, 0 tCK
1723                  */
1724                 p_pctl_timing->tmod = LPDDR2_tMOD;
1725                 /*
1726                  * tRSTL, 0 tCK
1727                  */
1728                 p_pctl_timing->trstl = LPDDR2_tRSTL;
1729                 /*
1730                  * tZQCL, 360ns
1731                  */
1732                 p_pctl_timing->tzqcl =
1733                     ((LPDDR2_tZQCL * nMHz + 999) / 1000) & 0x3FF;
1734                 /*
1735                  * tMRR, 2 tCK
1736                  */
1737                 p_pctl_timing->tmrr = LPDDR2_tMRR;
1738                 /*
1739                  * tCKESR, max(3tCK,15ns)
1740                  */
1741                 tmp = ((LPDDR2_tCKESR * nMHz + 999) / 1000);
1742                 if (tmp < 3) {
1743                         tmp = 3;
1744                 }
1745                 p_pctl_timing->tckesr = tmp & 0xF;
1746                 /*
1747                  * tDPD, 500us
1748                  */
1749                 p_pctl_timing->tdpd = LPDDR2_tDPD_US;
1750
1751                 /*************************************************
1752                 * NOC Timing
1753                 **************************************************/
1754                 p_noc_timing->b.BurstLen = ((bl >> 1) & 0x7);
1755         }
1756
1757 out:
1758         return ret;
1759 }
1760
1761 /*----------------------------------------------------------------------
1762 *Name    : uint32_t __sramlocalfunc ddr_update_timing(void)
1763 *Desc    : ¸üÐÂpctl phy Ïà¹Øtiming¼Ä´æÆ÷
1764 *Params  : void
1765 *Return  : 0 ³É¹¦
1766 *Notes   :
1767 *----------------------------------------------------------------------*/
1768 static uint32 __sramfunc ddr_update_timing(void)
1769 {
1770         uint32_t bl_tmp;
1771         uint32_t ret = 0;
1772
1773         PCTL_TIMING_T *p_pctl_timing = &(DATA(ddr_reg).pctl_timing);
1774         NOC_TIMING_T *p_noc_timing = &(DATA(ddr_reg).noc_timing);
1775
1776         ddr_copy((uint32 *)&(pDDR_Reg->TOGCNT1U),
1777                  (uint32 *)&(p_pctl_timing->togcnt1u), 34);
1778 /*    pPHY_Reg->PHY_REG1 |= PHY_Burst8;*/    /*ddr3 burst length¹Ì¶¨Îª8*/
1779         pPHY_Reg->PHY_REGb = ((p_pctl_timing->tcl << 4) | (p_pctl_timing->tal));
1780         pPHY_Reg->PHY_REGc = p_pctl_timing->tcwl;
1781         *(volatile uint32 *)SysSrv_DdrTiming = p_noc_timing->d32;
1782         /*Update PCTL BL*/
1783         if (DATA(ddr_reg).mem_type == DDR3) {
1784                 pDDR_Reg->MCFG =
1785                     (pDDR_Reg->MCFG &
1786                      (~(0x1 | (0x3 << 18) | (0x1 << 17) | (0x1 << 16))))
1787                     | ddr2_ddr3_bl_8 | tfaw_cfg(5) | pd_exit_slow | pd_type(1);
1788                 pDDR_Reg->DFITRDDATAEN = (pDDR_Reg->TAL + pDDR_Reg->TCL) - 3;   /*trdata_en = rl-3*/
1789                 pDDR_Reg->DFITPHYWRLAT = pDDR_Reg->TCWL - 1;
1790         } else if (DATA(ddr_reg).mem_type == LPDDR2) {
1791                 if ((DATA(ddr_reg).ddrMR[1] & 0x7) == LPDDR2_BL8) {
1792                         bl_tmp = mddr_lpddr2_bl_8;
1793                         pPHY_Reg->PHY_REG1 |= PHY_Burst8;
1794                 } else if ((DATA(ddr_reg).ddrMR[1] & 0x7) == LPDDR2_BL4) {
1795                         bl_tmp = mddr_lpddr2_bl_4;
1796                         pPHY_Reg->PHY_REG1 &= (~PHY_Burst8);
1797                 } else{         /*if((DATA(ddr_reg).ddrMR[1] & 0x7) == LPDDR2_BL16)*/
1798                         bl_tmp = mddr_lpddr2_bl_16;
1799                         ret = -1;
1800                 }
1801                 if (DATA(ddr_freq) >= 200) {
1802                         pDDR_Reg->MCFG =
1803                             (pDDR_Reg->MCFG &
1804                              (~
1805                               ((0x3 << 20) | (0x3 << 18) | (0x1 << 17) |
1806                                (0x1 << 16)))) | bl_tmp | tfaw_cfg(5) |
1807                             pd_exit_fast | pd_type(1);
1808                 } else {
1809                         pDDR_Reg->MCFG =
1810                             (pDDR_Reg->MCFG &
1811                              (~
1812                               ((0x3 << 20) | (0x3 << 18) | (0x1 << 17) |
1813                                (0x1 << 16)))) | mddr_lpddr2_bl_8 | tfaw_cfg(6) |
1814                             pd_exit_fast | pd_type(1);
1815                 }
1816                 pDDR_Reg->DFITRDDATAEN = pDDR_Reg->TCL - 3;
1817                 pDDR_Reg->DFITPHYWRLAT = pDDR_Reg->TCWL;
1818
1819         }
1820         return ret;
1821 }
1822
1823 /*----------------------------------------------------------------------
1824 *Name    : uint32_t __sramlocalfunc ddr_update_mr(void)
1825 *Desc    : ¸üÐÂ¿ÅÁ£MR¼Ä´æÆ÷
1826 *Params  : void
1827 *Return  : void
1828 *Notes   :
1829 *----------------------------------------------------------------------*/
1830 static uint32 __sramfunc ddr_update_mr(void)
1831 {
1832         /*uint32 cs;
1833         cs = READ_CS_INFO();
1834         cs = (1 << cs) - 1;     */
1835         if (DATA(ddr_reg).mem_type == DDR3) {
1836                 if (DATA(ddr_freq) > DDR3_DDR2_DLL_DISABLE_FREQ) {
1837                         if (DATA(ddr_dll_status) == DDR3_DLL_DISABLE) { /*off -> on*/
1838                                 ddr_send_command(3, MRS_cmd, bank_addr(0x1) | cmd_addr((DATA(ddr_reg).ddrMR[1])));      /*DLL enable*/
1839                                 ddr_send_command(3, MRS_cmd, bank_addr(0x0) | cmd_addr(((DATA(ddr_reg).ddrMR[0])) | DDR3_DLL_RESET));   /*DLL reset*/
1840                                 ddr_delayus(2); /*at least 200 DDR cycle*/
1841                                 ddr_send_command(3, MRS_cmd,
1842                                                  bank_addr(0x0) |
1843                                                  cmd_addr((DATA(ddr_reg).ddrMR
1844                                                            [0])));
1845                                 DATA(ddr_dll_status) = DDR3_DLL_ENABLE;
1846                         } else{         /*on -> on*/
1847                                 ddr_send_command(3, MRS_cmd,
1848                                                  bank_addr(0x1) |
1849                                                  cmd_addr((DATA(ddr_reg).ddrMR
1850                                                            [1])));
1851                                 ddr_send_command(3, MRS_cmd,
1852                                                  bank_addr(0x0) |
1853                                                  cmd_addr((DATA(ddr_reg).ddrMR
1854                                                            [0])));
1855                         }
1856                 } else {
1857                         ddr_send_command(3, MRS_cmd, bank_addr(0x1) | cmd_addr(((DATA(ddr_reg).ddrMR[1])) | DDR3_DLL_DISABLE)); /*DLL disable*/
1858                         ddr_send_command(3, MRS_cmd,
1859                                          bank_addr(0x0) |
1860                                          cmd_addr((DATA(ddr_reg).ddrMR[0])));
1861                         DATA(ddr_dll_status) = DDR3_DLL_DISABLE;
1862                 }
1863                 ddr_send_command(3, MRS_cmd,
1864                                  bank_addr(0x2) |
1865                                  cmd_addr((DATA(ddr_reg).ddrMR[2])));
1866         } else if (DATA(ddr_reg).mem_type == LPDDR2) {
1867                 ddr_send_command(3, MRS_cmd,
1868                                  lpddr2_ma(0x1) |
1869                                  lpddr2_op(DATA(ddr_reg).ddrMR[1]));
1870                 ddr_send_command(3, MRS_cmd,
1871                                  lpddr2_ma(0x2) |
1872                                  lpddr2_op(DATA(ddr_reg).ddrMR[2]));
1873                 ddr_send_command(3, MRS_cmd,
1874                                  lpddr2_ma(0x3) |
1875                                  lpddr2_op(DATA(ddr_reg).ddrMR[3]));
1876         }
1877         return 0;
1878 }
1879
1880 /*----------------------------------------------------------------------
1881 *Name    : void __sramlocalfunc ddr_update_odt(void)
1882 *Desc    : update PHY odt & PHY driver impedance
1883 *Params  : void
1884 *Return  : void
1885 *Notes   :-------------------------------------------------*/
1886 static void __sramfunc ddr_update_odt(void)
1887 {
1888         /*adjust DRV and ODT*/
1889         uint32 phy_odt;
1890         if (DATA(ddr_reg).mem_type == DDR3) {
1891                 if (DATA(ddr_freq) <= PHY_ODT_DISABLE_FREQ) {
1892                         phy_odt = PHY_RTT_DISABLE;
1893                 } else {
1894                         phy_odt = PHY_RTT_216ohm;
1895                 }
1896         } else {
1897                 phy_odt = PHY_RTT_DISABLE;
1898         }
1899
1900         pPHY_Reg->PHY_REG21 = PHY_DRV_ODT_SET(phy_odt); /*DQS0 odt*/
1901         pPHY_Reg->PHY_REG31 = PHY_DRV_ODT_SET(phy_odt); /*DQS1 odt*/
1902         pPHY_Reg->PHY_REG41 = PHY_DRV_ODT_SET(phy_odt); /*DQS2 odt*/
1903         pPHY_Reg->PHY_REG51 = PHY_DRV_ODT_SET(phy_odt); /*DQS3 odt*/
1904
1905         pPHY_Reg->PHY_REG11 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*cmd drv*/
1906         pPHY_Reg->PHY_REG16 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*clk drv*/
1907
1908         pPHY_Reg->PHY_REG20 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*DQS0 drv*/
1909         pPHY_Reg->PHY_REG30 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*DQS1 drv*/
1910         pPHY_Reg->PHY_REG40 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*DQS2 drv*/
1911         pPHY_Reg->PHY_REG50 = PHY_DRV_ODT_SET(PHY_RON_44ohm);   /*DQS3 drv*/
1912
1913         dsb();
1914 }
1915
1916 #if 0
1917 void PIE_FUNC(ddr_adjust_config)(void)
1918 {
1919         /*enter config state*/
1920         ddr_move_to_Config_state();
1921
1922         /*set auto power down idle*/
1923         pDDR_Reg->MCFG = (pDDR_Reg->MCFG & 0xffff00ff) | (PD_IDLE << 8);
1924         /*enable the hardware low-power interface*/
1925         pDDR_Reg->SCFG.b.hw_low_power_en = 1;
1926         ddr_update_odt();
1927         /*enter access state*/
1928         ddr_move_to_Access_state();
1929 }
1930
1931 EXPORT_PIE_SYMBOL(FUNC(ddr_adjust_config));
1932
1933 /*----------------------------------------------------------------------
1934 Name    : __sramfunc void ddr_adjust_config(uint32_t dram_type)
1935 Desc    :
1936 Params  : dram_type ->¿ÅÁ£ÀàÐÍ
1937 Return  : void
1938 Notes   :
1939 ----------------------------------------------------------------------*/
1940
1941 static void ddr_adjust_config(uint32_t dram_type)
1942 {
1943         unsigned long save_sp;
1944         uint32 i;
1945         volatile uint32 n;
1946         volatile unsigned int *temp = (volatile unsigned int *)SRAM_CODE_OFFSET;
1947
1948     /** 1. Make sure there is no host access */
1949         flush_cache_all();
1950         outer_flush_all();
1951         flush_tlb_all();
1952         for (i = 0; i < 2; i++) {
1953                 n = temp[1024 * i];
1954                 barrier();
1955         }
1956         n = pDDR_Reg->SCFG.d32;
1957         n = pPHY_Reg->PHY_REG1;
1958         n = pCRU_Reg->CRU_PLL_CON[0][0];
1959         n = *(volatile uint32_t *)SysSrv_DdrTiming;
1960         dsb();
1961
1962         call_with_stack(fn_to_pie(rockchip_pie_chunk, &FUNC(ddr_adjust_config)),
1963                         (void *)0, rockchip_sram_stack);
1964
1965 }
1966 #endif
1967
1968 static void __sramfunc idle_port(void)
1969 {
1970         int i;
1971         uint32 clk_gate[11];
1972         uint32 pd_status;
1973         uint32 idle_req, idle_stus;
1974         /*bit0:core bit1:gpu bit2:video bit3:vio 1:power off*/
1975         pd_status = *(volatile uint32 *)PMU_PWEDN_ST;
1976         /*save clock gate status*/
1977         for (i = 0; i < 11; i++) {
1978                 clk_gate[i] = pCRU_Reg->CRU_CLKGATE_CON[i];
1979         }
1980         /*enable all clock gate for request idle*/
1981         for (i = 0; i < 11; i++) {
1982                 pCRU_Reg->CRU_CLKGATE_CON[i] = 0xffff0000;
1983         }
1984
1985         idle_req = (1 << peri_pwr_idlereq); /*peri*/
1986         idle_stus = peri_pwr_idle;
1987         if ((pd_status & (0x1<<3)) == 0) {/*pwr_vio*/
1988                 idle_req |= (1 << vio_pwr_idlereq);
1989                 idle_stus |= vio_pwr_idle;
1990         }
1991         if ((pd_status & (0x1<<2)) == 0) {/*pwr_vpu*/
1992                 idle_req |= (1 << vpu_pwr_idlereq);
1993                 idle_stus |= vpu_pwr_idle;
1994         }
1995         if ((pd_status & (0x1<<1)) == 0) {/*pwr_gpu*/
1996                 idle_req |= (1 << gpu_pwr_idlereq);
1997                 idle_stus |= gpu_pwr_idle;
1998         }
1999
2000         pGRF_Reg->GRF_SOC_CON[2] = (idle_req << 16) | idle_req;
2001         dsb();
2002         while ((pGRF_Reg->GRF_SOC_STATUS0 & idle_stus) != idle_stus)
2003         ;
2004
2005         /*resume clock gate status*/
2006         for (i = 0; i < 11; i++)
2007                 pCRU_Reg->CRU_CLKGATE_CON[i] = (clk_gate[i] | 0xffff0000);
2008 }
2009
2010 static void __sramfunc deidle_port(void)
2011 {
2012         int i;
2013         uint32 clk_gate[11];
2014         uint32 pd_status;
2015         uint32 idle_req, idle_stus;
2016         /*bit0:core bit1:gpu bit2:video bit3:vio 1:power off*/
2017         pd_status = *(volatile uint32 *)PMU_PWEDN_ST;
2018
2019         /*save clock gate status*/
2020         for (i = 0; i < 11; i++) {
2021                 clk_gate[i] = pCRU_Reg->CRU_CLKGATE_CON[i];
2022         }
2023         /*enable all clock gate for request idle*/
2024         for (i = 0; i < 11; i++) {
2025                 pCRU_Reg->CRU_CLKGATE_CON[i] = 0xffff0000;
2026         }
2027
2028         idle_req = (1 << peri_pwr_idlereq); /*peri*/
2029         idle_stus = peri_pwr_idle;
2030         if ((pd_status & (0x1<<3)) == 0) {/*pwr_vio*/
2031                 idle_req |= (1 << vio_pwr_idlereq);
2032                 idle_stus |= vio_pwr_idle;
2033         }
2034         if ((pd_status & (0x1<<2)) == 0) {/*pwr_vpu*/
2035                 idle_req |= (1 << vpu_pwr_idlereq);
2036                 idle_stus |= vpu_pwr_idle;
2037         }
2038         if ((pd_status & (0x1<<1)) == 0) {/*pwr_gpu*/
2039                 idle_req |= (1 << gpu_pwr_idlereq);
2040                 idle_stus |= gpu_pwr_idle;
2041         }
2042
2043         pGRF_Reg->GRF_SOC_CON[2] = (idle_req << 16) | 0 ;
2044         dsb();
2045         while ((pGRF_Reg->GRF_SOC_STATUS0 & idle_stus) != 0)
2046         ;
2047
2048         /*resume clock gate status*/
2049         for (i = 0; i < 11; i++)
2050                 pCRU_Reg->CRU_CLKGATE_CON[i] = (clk_gate[i] | 0xffff0000);
2051
2052 }
2053
2054 /*----------------------------------------------------------------------
2055 *Name    : void __sramlocalfunc ddr_selfrefresh_enter(uint32 nMHz)
2056 *Desc    : ½øÈë×ÔË¢ÐÂ
2057 *Params  : nMHz ->ddrÆµÂÊ
2058 *Return  : void
2059 *Notes   :
2060 *----------------------------------------------------------------------*/
2061 #if 1
2062 static void __sramfunc ddr_selfrefresh_enter(uint32 nMHz)
2063 {
2064         ddr_move_to_Config_state();
2065         ddr_move_to_Lowpower_state();
2066         pGRF_Reg->GRF_SOC_CON[2] = GRF_DDR_LP_EN;
2067         pPHY_Reg->PHY_REG0 = (pPHY_Reg->PHY_REG0 & (~(0x3 << 2)));      /*phy soft reset*/
2068         dsb();
2069         pCRU_Reg->CRU_CLKGATE_CON[0] = ((0x1 << 2) << 16) | (1 << 2);   /*disable DDR PHY clock*/
2070         ddr_delayus(1);
2071 }
2072 #endif
2073
2074 static uint32 dtt_buffer[8];
2075
2076 /*----------------------------------------------------------------------
2077 *Name    : void ddr_dtt_check(void)
2078 *Desc    : data training check
2079 *Params  : void
2080 *Return  : void
2081 *Notes   :
2082 *----------------------------------------------------------------------*/
2083 static void ddr_dtt_check(void)
2084 {
2085         uint32 i;
2086         for (i = 0; i < 8; i++) {
2087                 dtt_buffer[i] = p_copy_data[i];
2088         }
2089         dsb();
2090         flush_cache_all();
2091         outer_flush_all();
2092         for (i = 0; i < 8; i++) {
2093                 if (dtt_buffer[i] != p_copy_data[i]) {
2094 /*            sram_printascii("DTT failed!\n");*/
2095                         break;
2096                 }
2097                 dtt_buffer[i] = 0;
2098         }
2099 }
2100
2101 /*----------------------------------------------------------------------
2102 Name    : void __sramlocalfunc ddr_selfrefresh_exit(void)
2103 Desc    : ÍË³ö×ÔË¢ÐÂ
2104 Params  : void
2105 Return  : void
2106 Notes   :
2107 ----------------------------------------------------------------------*/
2108 #if 1
2109 static void __sramfunc ddr_selfrefresh_exit(void)
2110 {
2111         pCRU_Reg->CRU_CLKGATE_CON[0] = ((0x1 << 2) << 16) | (0 << 2);   /*enable DDR PHY clock*/
2112         dsb();
2113         ddr_delayus(1);
2114         pPHY_Reg->PHY_REG0 |= (1 << 2); /*soft de-reset analogue(dll)*/
2115         ddr_delayus(5);
2116         pPHY_Reg->PHY_REG0 |= (1 << 3);/*soft de-reset digital*/
2117         pGRF_Reg->GRF_SOC_CON[2] = GRF_DDR_LP_DISB;
2118         /*pPHY_Reg->PHY_REG264 |= (1<<1);*/
2119         dsb();
2120         ddr_move_to_Config_state();
2121         ddr_data_training();
2122         ddr_move_to_Access_state();
2123 /*    ddr_dtt_check();*/
2124 }
2125
2126 #endif
2127 /*----------------------------------------------------------------------
2128 *Name    : void __sramlocalfunc ddr_change_freq_in(uint32 freq_slew)
2129 *Desc    : ÉèÖÃddr pllÇ°µÄtiming¼°mr²ÎÊýµ÷Õû
2130 *Params  : freq_slew :±äÆµÐ±ÂÊ 1ÉýÆ½  0½µÆµ
2131 *Return  : void
2132 *Notes   :
2133 *----------------------------------------------------------------------*/
2134 static void __sramlocalfunc ddr_change_freq_in(uint32 freq_slew)
2135 {
2136         uint32 value_100n, value_1u;
2137
2138         if (freq_slew == 1) {
2139                 value_100n = DATA(ddr_reg).pctl_timing.togcnt100n;
2140                 value_1u = DATA(ddr_reg).pctl_timing.togcnt1u;
2141                 DATA(ddr_reg).pctl_timing.togcnt1u = pDDR_Reg->TOGCNT1U;
2142                 DATA(ddr_reg).pctl_timing.togcnt100n = pDDR_Reg->TOGCNT100N;
2143                 ddr_update_timing();
2144                 ddr_update_mr();
2145                 DATA(ddr_reg).pctl_timing.togcnt100n = value_100n;
2146                 DATA(ddr_reg).pctl_timing.togcnt1u = value_1u;
2147         } else {
2148                 pDDR_Reg->TOGCNT100N = DATA(ddr_reg).pctl_timing.togcnt100n;
2149                 pDDR_Reg->TOGCNT1U = DATA(ddr_reg).pctl_timing.togcnt1u;
2150         }
2151
2152         pDDR_Reg->TZQCSI = 0;
2153
2154 }
2155
2156 /*----------------------------------------------------------------------
2157 *Name    : void __sramlocalfunc ddr_change_freq_out(uint32 freq_slew)
2158 *Desc    : ÉèÖÃddr pllºóµÄtiming¼°mr²ÎÊýµ÷Õû
2159 *Params  : freq_slew :±äÆµÐ±ÂÊ 1ÉýÆ½  0½µÆµ
2160 *Return  : void
2161 *Notes   :
2162 *----------------------------------------------------------------------*/
2163 static void __sramlocalfunc ddr_change_freq_out(uint32 freq_slew)
2164 {
2165         if (freq_slew == 1) {
2166                 pDDR_Reg->TOGCNT100N = DATA(ddr_reg).pctl_timing.togcnt100n;
2167                 pDDR_Reg->TOGCNT1U = DATA(ddr_reg).pctl_timing.togcnt1u;
2168                 pDDR_Reg->TZQCSI = DATA(ddr_reg).pctl_timing.tzqcsi;
2169         } else {
2170                 ddr_update_timing();
2171                 ddr_update_mr();
2172         }
2173         ddr_data_training();
2174 }
2175
2176 static void __sramfunc ddr_SRE_2_SRX(uint32 freq, uint32 freq_slew)
2177 {
2178         idle_port();
2179
2180         ddr_move_to_Config_state();
2181         DATA(ddr_freq) = freq;
2182         ddr_change_freq_in(freq_slew);
2183         ddr_move_to_Lowpower_state();
2184         pGRF_Reg->GRF_SOC_CON[2] = GRF_DDR_LP_EN;
2185         pPHY_Reg->PHY_REG0 = (pPHY_Reg->PHY_REG0 & (~(0x3 << 2)));      /*phy soft reset*/
2186         dsb();
2187     /* 3. change frequence  */
2188         FUNC(ddr_set_pll) (freq, 1);
2189         ddr_set_dll_bypass(freq);       /*set phy dll mode;*/
2190         /*pPHY_Reg->PHY_REG0 = (pPHY_Reg->PHY_REG0 | (0x3 << 2)); */      /*phy soft de-reset */
2191         pPHY_Reg->PHY_REG0 |= (1 << 2); /*soft de-reset analogue(dll)*/
2192         ddr_delayus(5);
2193         pPHY_Reg->PHY_REG0 |= (1 << 3);/*soft de-reset digital*/
2194         pGRF_Reg->GRF_SOC_CON[2] = GRF_DDR_LP_DISB;
2195         dsb();
2196         ddr_update_odt();
2197         ddr_move_to_Config_state();
2198         ddr_change_freq_out(freq_slew);
2199         ddr_move_to_Access_state();
2200
2201         deidle_port();
2202 }
2203
2204 void PIE_FUNC(ddr_change_freq_sram)(void *arg)
2205 {
2206         struct ddr_change_freq_sram_param *param = arg;
2207         /* Make sure ddr_SRE_2_SRX paramter less than 4 */
2208         ddr_SRE_2_SRX(param->freq, param->freq_slew);
2209 }
2210
2211 EXPORT_PIE_SYMBOL(FUNC(ddr_change_freq_sram));
2212
2213 typedef struct freq_tag {
2214         uint32_t nMHz;
2215         struct ddr_freq_t *p_ddr_freq_t;
2216 } freq_t;
2217
2218 /*----------------------------------------------------------------------
2219 *Name    : uint32_t __sramfunc ddr_change_freq(uint32_t nMHz)
2220 *Desc    : ddr±äÆµ
2221 *Params  : nMHz -> ±äÆµµÄÆµÂÊÖµ
2222 *Return  : ÆµÂÊÖµ
2223 *Notes   :
2224 *----------------------------------------------------------------------*/
2225 static uint32 ddr_change_freq_sram(void *arg)
2226 {
2227         uint32 ret;
2228         uint32 i;
2229         volatile uint32 n;
2230         unsigned long flags;
2231         volatile unsigned int *temp = (volatile unsigned int *)SRAM_CODE_OFFSET;
2232         freq_t *p_freq_t = (freq_t *) arg;
2233         uint32 nMHz = p_freq_t->nMHz;
2234 #if defined (DDR_CHANGE_FREQ_IN_LCDC_VSYNC)
2235         struct ddr_freq_t *p_ddr_freq_t = p_freq_t->p_ddr_freq_t;
2236 #endif
2237
2238         struct ddr_change_freq_sram_param param;
2239         /*uint32 freq;*/
2240         uint32 freq_slew;
2241         uint32 arm_freq;
2242         arm_freq = ddr_get_pll_freq(APLL);
2243         *kern_to_pie(rockchip_pie_chunk, &DATA(loops_per_us)) =
2244             LPJ_100MHZ * arm_freq / 1000000;
2245         ret = p_ddr_set_pll(nMHz, 0);
2246         if (ret == *p_ddr_freq) {
2247                 goto out;
2248         } else {
2249                 freq_slew = (ret > *p_ddr_freq) ? 1 : -1;
2250         }
2251         ddr_get_parameter(ret);
2252         /*kern_to_pie(rockchip_pie_chunk, &DATA(ddr_freq))= ret;*/
2253     /** 1. Make sure there is no host access */
2254         local_irq_save(flags);
2255         local_fiq_disable();
2256         flush_cache_all();
2257         outer_flush_all();
2258         flush_tlb_all();
2259
2260 #if defined (DDR_CHANGE_FREQ_IN_LCDC_VSYNC)
2261         if (p_ddr_freq_t->screen_ft_us > 0) {
2262                 p_ddr_freq_t->t1 = cpu_clock(0);
2263                 p_ddr_freq_t->t2 = (uint32)(p_ddr_freq_t->t1 - p_ddr_freq_t->t0);   /*ns*/
2264
2265                 if ((p_ddr_freq_t->t2 > p_ddr_freq_t->screen_ft_us*1000) && (p_ddr_freq_t->screen_ft_us != 0xfefefefe)) {
2266                         ret = 0;
2267                         goto end;
2268                 } else {
2269                         rk_fb_poll_wait_frame_complete();
2270                 }
2271         }
2272 #endif
2273     /*8KB SRAM*/
2274         for (i = 0; i < 2; i++) {
2275                 n = temp[1024 * i];
2276                 barrier();
2277         }
2278         n = pDDR_Reg->SCFG.d32;
2279         n = pPHY_Reg->PHY_REG1;
2280         n = pCRU_Reg->CRU_PLL_CON[0][0];
2281         n = *(volatile uint32_t *)SysSrv_DdrTiming;
2282         n = pGRF_Reg->GRF_SOC_STATUS0;
2283         dsb();
2284         param.freq = ret;
2285         param.freq_slew = freq_slew;
2286         call_with_stack(fn_to_pie
2287                         (rockchip_pie_chunk, &FUNC(ddr_change_freq_sram)),
2288                         &param,
2289                         rockchip_sram_stack - (NR_CPUS -
2290                                                1) * PAUSE_CPU_STACK_SIZE);
2291     /** 5. Issues a Mode Exit command   */
2292         ddr_dtt_check();
2293 #if defined (DDR_CHANGE_FREQ_IN_LCDC_VSYNC)
2294 end:
2295 #endif
2296         local_fiq_enable();
2297         local_irq_restore(flags);
2298 /*    clk_set_rate(clk_get(NULL, "ddr_pll"), 0);    */
2299 out:
2300         return ret;
2301 }
2302
2303 bool DEFINE_PIE_DATA(cpu_pause[NR_CPUS]);
2304 volatile bool *DATA(p_cpu_pause);
2305 static inline bool is_cpu0_paused(unsigned int cpu)
2306 {
2307         smp_rmb();
2308         return DATA(cpu_pause)[0];
2309 }
2310
2311 static inline void set_cpuX_paused(unsigned int cpu, bool pause)
2312 {
2313         DATA(cpu_pause)[cpu] = pause;
2314         smp_wmb();
2315 }
2316
2317 static inline bool is_cpuX_paused(unsigned int cpu)
2318 {
2319         smp_rmb();
2320         return DATA(p_cpu_pause)[cpu];
2321 }
2322
2323 static inline void set_cpu0_paused(bool pause)
2324 {
2325         DATA(p_cpu_pause)[0] = pause;
2326         smp_wmb();
2327 }
2328
2329 #define MAX_TIMEOUT (16000000UL << 6)   /*>0.64s*/
2330
2331 /* Do not use stack, safe on SMP */
2332 void PIE_FUNC(_pause_cpu)(void *arg)
2333 {
2334         unsigned int cpu = (unsigned int)arg;
2335
2336         set_cpuX_paused(cpu, true);
2337         while (is_cpu0_paused(cpu))
2338     ;
2339         set_cpuX_paused(cpu, false);
2340 }
2341
2342 static void pause_cpu(void *info)
2343 {
2344         unsigned int cpu = raw_smp_processor_id();
2345
2346         call_with_stack(fn_to_pie(rockchip_pie_chunk, &FUNC(_pause_cpu)),
2347                         (void *)cpu,
2348                         rockchip_sram_stack - (cpu - 1) * PAUSE_CPU_STACK_SIZE);
2349 }
2350
2351 static void wait_cpu(void *info)
2352 {
2353 }
2354
2355 static int call_with_single_cpu(u32(*fn) (void *arg), void *arg)
2356 {
2357         u32 timeout = MAX_TIMEOUT;
2358         unsigned int cpu;
2359         unsigned int this_cpu = smp_processor_id();     /*»ñÈ¡µ±Ç°cpu*/
2360         int ret = 0;
2361         cpu_maps_update_begin();
2362         local_bh_disable();     /*disable swi*/
2363         set_cpu0_paused(true);
2364         smp_call_function((smp_call_func_t) pause_cpu, NULL, 0);
2365
2366         for_each_online_cpu(cpu) {
2367                 if (cpu == this_cpu)
2368                         continue;
2369                 while (!is_cpuX_paused(cpu) && --timeout)
2370                     ;
2371                 if (timeout == 0) {
2372                         pr_err("pause cpu %d timeout\n", cpu);
2373                         goto out;
2374                 }
2375         }
2376
2377         ret = fn(arg);
2378
2379 out:
2380         set_cpu0_paused(false);
2381         local_bh_enable();
2382         smp_call_function(wait_cpu, NULL, true);
2383         cpu_maps_update_done();
2384
2385         return ret;
2386 }
2387
2388 static int __ddr_change_freq(uint32_t nMHz, struct ddr_freq_t ddr_freq_t)
2389 {
2390         freq_t freq;
2391         int ret = 0;
2392
2393         freq.nMHz = nMHz;
2394         freq.p_ddr_freq_t = &ddr_freq_t;
2395         ret = call_with_single_cpu(&ddr_change_freq_sram, (void *)&freq);
2396         /*ret = ddr_change_freq_sram((void*)&freq);*/
2397         return ret;
2398 }
2399
2400 static int _ddr_change_freq(uint32 nMHz)
2401 {
2402         struct ddr_freq_t ddr_freq_t;
2403 #if defined (DDR_CHANGE_FREQ_IN_LCDC_VSYNC)
2404         unsigned long remain_t, vblank_t, pass_t;
2405         static unsigned long reserve_t = 800;   /*us*/
2406         unsigned long long tmp;
2407         int test_count = 0;
2408 #endif
2409         int ret;
2410         /*ddr_print("ddr change freq to:%d\n",nMHz);*/
2411         memset(&ddr_freq_t, 0x00, sizeof(ddr_freq_t));
2412
2413 #if defined (DDR_CHANGE_FREQ_IN_LCDC_VSYNC)
2414         do {
2415                 ddr_freq_t.screen_ft_us = rk_fb_get_prmry_screen_ft();
2416                 ddr_freq_t.t0 = rk_fb_get_prmry_screen_framedone_t();
2417                 if (!ddr_freq_t.screen_ft_us)
2418                         return __ddr_change_freq(nMHz, ddr_freq_t);
2419
2420                 tmp = cpu_clock(0) - ddr_freq_t.t0;
2421                 do_div(tmp, 1000);
2422                 pass_t = tmp;
2423                 /*lost frame interrupt*/
2424                 while (pass_t > ddr_freq_t.screen_ft_us) {
2425                         int n = pass_t / ddr_freq_t.screen_ft_us;
2426
2427                         /*printk("lost frame int, pass_t:%lu\n", pass_t);*/
2428                         pass_t -= n * ddr_freq_t.screen_ft_us;
2429                         ddr_freq_t.t0 += n * ddr_freq_t.screen_ft_us * 1000;
2430                 }
2431
2432                 remain_t = ddr_freq_t.screen_ft_us - pass_t;
2433                 if (remain_t < reserve_t) {
2434                         /*printk("remain_t(%lu) < reserve_t(%lu)\n", remain_t, reserve_t);*/
2435                         vblank_t = rk_fb_get_prmry_screen_vbt();
2436                         usleep_range(remain_t + vblank_t, remain_t + vblank_t);
2437                         continue;
2438                 }
2439                 /*test 10 times*/
2440                 test_count++;
2441                 if (test_count > 10) {
2442                         ddr_freq_t.screen_ft_us = 0xfefefefe;
2443                 }
2444                 /*printk("ft:%lu, pass_t:%lu, remaint_t:%lu, reservet_t:%lu\n",
2445                  *     ddr_freq_t.screen_ft_us, (unsigned long)pass_t, remain_t, reserve_t);*/
2446                 usleep_range(remain_t - reserve_t, remain_t - reserve_t);
2447                 flush_tlb_all();
2448
2449                 ret = __ddr_change_freq(nMHz, ddr_freq_t);
2450                 if (ret) {
2451                         reserve_t = 800;
2452                         return ret;
2453                 } else {
2454                         if (reserve_t < 3000)
2455                                 reserve_t += 200;
2456                 }
2457         } while (1);
2458 #else
2459         ret = __ddr_change_freq(nMHz, ddr_freq_t);
2460 #endif
2461
2462         return ret;
2463 }
2464
2465 /*----------------------------------------------------------------------
2466 *Name    : void ddr_set_auto_self_refresh(bool en)
2467 *Desc    : ÉèÖÃ½øÈë selfrefesh µÄÖÜÆÚÊý
2468 *Params  : en -> Ê¹ÄÜauto selfrefresh
2469 *Return  : ÆµÂÊÖµ
2470 *Notes   : ÖÜÆÚÊýÎª1*32 cycle
2471 *----------------------------------------------------------------------*/
2472 static void _ddr_set_auto_self_refresh(bool en)
2473 {
2474         /*set auto self-refresh idle    */
2475         *kern_to_pie(rockchip_pie_chunk, &DATA(ddr_sr_idle)) = en ? SR_IDLE : 0;
2476 }
2477
2478 /*----------------------------------------------------------------------
2479 *Name    : void __sramfunc ddr_suspend(void)
2480 *Desc    : ½øÈëddr suspend
2481 *Params  : void
2482 *Return  : void
2483 *Notes   :
2484 *----------------------------------------------------------------------*/
2485 void PIE_FUNC(ddr_suspend)(void)
2486 {
2487         ddr_selfrefresh_enter(0);
2488         pCRU_Reg->CRU_MODE_CON = (0x1 << ((1 * 4) + 16)) | (0x0 << (1 * 4));    /*PLL slow-mode*/
2489         dsb();
2490         ddr_delayus(1);
2491         pCRU_Reg->CRU_PLL_CON[1][1] = ((0x1 << 13) << 16) | (0x1 << 13);        /*PLL power-down*/
2492         dsb();
2493         ddr_delayus(1);
2494
2495 }
2496
2497 EXPORT_PIE_SYMBOL(FUNC(ddr_suspend));
2498
2499 #if 0
2500 static void ddr_suspend(void)
2501 {
2502         uint32 i;
2503         volatile uint32 n;
2504         volatile unsigned int *temp = (volatile unsigned int *)SRAM_CODE_OFFSET;
2505     /** 1. Make sure there is no host access */
2506         flush_cache_all();
2507         outer_flush_all();
2508     /*flush_tlb_all();*/
2509
2510     /*sram size = 8KB*/
2511         for (i = 0; i < 2; i++) {
2512                 n = temp[1024 * i];
2513                 barrier();
2514         }
2515         n = pDDR_Reg->SCFG.d32;
2516         n = pPHY_Reg->PHY_REG1;
2517         n = pCRU_Reg->CRU_PLL_CON[0][0];
2518         n = *(volatile uint32_t *)SysSrv_DdrTiming;
2519         n = pGRF_Reg->GRF_SOC_STATUS0;
2520         dsb();
2521
2522         fn_to_pie(rockchip_pie_chunk, &FUNC(ddr_suspend)) ();
2523 }
2524 #endif
2525
2526 /*----------------------------------------------------------------------
2527 *Name    : void __sramfunc ddr_resume(void)
2528 *Desc    : ddr resume
2529 *Params  : void
2530 *Return  : void
2531 *Notes   :
2532 *----------------------------------------------------------------------*/
2533 void PIE_FUNC(ddr_resume)(void)
2534 {
2535         uint32 delay = 1000;
2536
2537         pCRU_Reg->CRU_PLL_CON[1][1] = ((0x1 << 13) << 16) | (0x0 << 13);        /*PLL no power-down*/
2538         dsb();
2539         while (delay > 0) {
2540                 ddr_delayus(1);
2541                 if (pCRU_Reg->CRU_PLL_CON[1][1] & (0x1 << 10))
2542                         break;
2543                 delay--;
2544         }
2545
2546         pCRU_Reg->CRU_MODE_CON = (0x1 << ((1 * 4) + 16)) | (0x1 << (1 * 4));    /*PLL normal*/
2547         dsb();
2548
2549         ddr_selfrefresh_exit();
2550 }
2551
2552 EXPORT_PIE_SYMBOL(FUNC(ddr_resume));
2553
2554 #if 0
2555 static void ddr_resume(void)
2556 {
2557         fn_to_pie(rockchip_pie_chunk, &FUNC(ddr_resume)) ();
2558 }
2559 #endif
2560
2561 /*----------------------------------------------------------------------
2562 *Name    : uint32 ddr_get_cap(void)
2563 *Desc    : »ñÈ¡ÈÝÁ¿£¬·µ»Ø×Ö½ÚÊý
2564 *Params  : void
2565 *Return  : ¿ÅÁ£ÈÝÁ¿
2566 *Notes   :
2567 *----------------------------------------------------------------------*/
2568 static uint32 ddr_get_cap(void)
2569 {
2570         uint32 cs, bank, row, col, row1, bw;
2571
2572         bank = READ_BK_INFO();
2573         row = READ_CS0_ROW_INFO();
2574         col = READ_COL_INFO();
2575         cs = READ_CS_INFO();
2576         bw = READ_BW_INFO();
2577         if (cs > 1) {
2578                 row1 = READ_CS1_ROW_INFO();
2579                 return ((1 << (row + col + bank + bw)) +
2580                         (1 << (row1 + col + bank + bw)));
2581         } else {
2582                 return (1 << (row + col + bank + bw));
2583         }
2584 }
2585
2586 static long _ddr_round_rate(uint32 nMHz)
2587 {
2588         return p_ddr_set_pll(nMHz, 0);
2589 }
2590
2591 enum ddr_bandwidth_id{
2592         ddrbw_wr_num = 0,
2593         ddrbw_rd_num,
2594         ddrbw_act_num,
2595         ddrbw_time_num,
2596         ddrbw_id_end
2597 };
2598
2599 static void ddr_dfi_monitor_strat(void)
2600 {
2601         pGRF_Reg->GRF_SOC_CON[0] = DDR_MONITOR_EN;
2602 }
2603 static void ddr_dfi_monitor_stop(void)
2604 {
2605         pGRF_Reg->GRF_SOC_CON[0] = DDR_MONITOR_DISB;
2606 }
2607
2608 static void _ddr_bandwidth_get(struct ddr_bw_info *ddr_bw_ch0, struct ddr_bw_info *ddr_bw_ch1)
2609 {
2610         uint32 ddr_bw_val[ddrbw_id_end], ddr_freq;
2611         u64 temp64;
2612         uint32 i;
2613         uint32 ddr_bw;
2614         uint32 bl;
2615
2616         if (p_ddr_reg->mem_type == DDR3)
2617                 bl = 8;
2618         else
2619                 bl = ((pDDR_Reg->MCFG >> 20) & 0x3) << 2;
2620
2621         ddr_bw = READ_BW_INFO();
2622         ddr_dfi_monitor_stop();
2623         for (i = 0; i < ddrbw_id_end; i++) {
2624                 ddr_bw_val[i] = *(uint32 *)(&(pGRF_Reg->GRF_DFI_WRNUM) + i);
2625         }
2626         if (!ddr_bw_val[ddrbw_time_num])
2627                 goto end;
2628
2629         ddr_freq = pDDR_Reg->TOGCNT1U;
2630         temp64 = ((u64)ddr_bw_val[ddrbw_wr_num] + (u64)ddr_bw_val[ddrbw_rd_num])
2631                  * bl / 2 * 100;
2632         do_div(temp64, ddr_bw_val[ddrbw_time_num]);
2633
2634         ddr_bw_ch0->ddr_percent = (uint32)temp64;
2635         ddr_bw_ch0->ddr_time = ddr_bw_val[ddrbw_time_num] / (ddr_freq * 1000);
2636         temp64 = ((u64)ddr_bw_val[ddrbw_wr_num] * bl * ddr_bw * 2) * ddr_freq;
2637         do_div(temp64, ddr_bw_val[ddrbw_time_num]);
2638         ddr_bw_ch0->ddr_wr = (uint32)temp64;
2639         temp64 = ((u64)ddr_bw_val[ddrbw_rd_num] * bl * ddr_bw * 2) * ddr_freq;
2640         do_div(temp64, ddr_bw_val[ddrbw_time_num]);
2641         ddr_bw_ch0->ddr_rd = (uint32)temp64;
2642         ddr_bw_ch0->ddr_act = ddr_bw_val[ddrbw_act_num];
2643         ddr_bw_ch0->ddr_total = ddr_freq * 2 * ddr_bw * 2;
2644 end:
2645         ddr_dfi_monitor_strat();
2646 }
2647
2648 /*----------------------------------------------------------------------
2649 *Name    : int ddr_init(uint32_t dram_speed_bin, uint32_t freq)
2650 *Desc    : ddr  ³õÊ¼»¯º¯Êý
2651 *Params  : dram_speed_bin ->ddr¿ÅÁ£ÀàÐÍ
2652 *          freq ->ÆµÂÊÖµ
2653 *Return  : 0 ³É¹¦
2654 *Notes   :
2655 *----------------------------------------------------------------------*/
2656 static int ddr_init(uint32_t dram_speed_bin, uint32 freq)
2657 {
2658         uint32_t value = 0;
2659         uint32_t cs, die = 1;
2660         /*uint32_t calStatusLeft, calStatusRight*/
2661         struct clk *clk;
2662
2663         ddr_print("version 1.02 20140828\n");
2664         cs = READ_CS_INFO();    /*case 1:1rank ; case 2:2rank*/
2665
2666         p_ddr_reg = kern_to_pie(rockchip_pie_chunk, &DATA(ddr_reg));
2667         p_ddr_freq = kern_to_pie(rockchip_pie_chunk, &DATA(ddr_freq));
2668         p_ddr_set_pll = fn_to_pie(rockchip_pie_chunk, &FUNC(ddr_set_pll));
2669         DATA(p_cpu_pause) =
2670             kern_to_pie(rockchip_pie_chunk, &DATA(cpu_pause[0]));
2671         p_ddr_reg->mem_type = ((pGRF_Reg->GRF_OS_REG[1] >> 13) & 0x7);
2672         p_ddr_reg->ddr_speed_bin = dram_speed_bin;
2673         *p_ddr_freq = 0;
2674         *kern_to_pie(rockchip_pie_chunk, &DATA(ddr_sr_idle)) = 0;
2675         *kern_to_pie(rockchip_pie_chunk, &DATA(ddr_dll_status)) =
2676             DDR3_DLL_DISABLE;
2677         p_copy_data = kern_to_pie(rockchip_pie_chunk, &copy_data[0]);
2678
2679         switch (p_ddr_reg->mem_type) {
2680         case DDR3:
2681                 ddr_print("DRAM Type:DDR3\n");
2682                 break;
2683         case LPDDR2:
2684                 ddr_print("DRAM Type:LPDDR2\n");
2685                 break;
2686         default:
2687                 ddr_print("ddr type error type=%d\n", (p_ddr_reg->mem_type));
2688         }
2689         die = 1 << (READ_BW_INFO() - READ_DIE_BW_INFO());
2690         p_ddr_reg->ddr_capability_per_die = ddr_get_cap() / (cs * die);
2691         ddr_print("%d CS, ROW=%d, Bank=%d, COL=%d, Total Capability=%dMB\n",
2692                                                 cs, READ_CS0_ROW_INFO(),
2693                                                 (0x1 << (READ_BK_INFO())),
2694                                                 READ_COL_INFO(),
2695                                                 (ddr_get_cap() >> 20));
2696
2697         clk = clk_get(NULL, "clk_ddr");
2698         if (IS_ERR(clk)) {
2699                 ddr_print("failed to get ddr clk\n");
2700                 clk = NULL;
2701         }
2702         if (freq != 0)
2703                 value = clk_set_rate(clk, 1000 * 1000 * freq);
2704         else
2705                 value = clk_set_rate(clk, clk_get_rate(clk));
2706         ddr_print("init success!!! freq=%luMHz\n",
2707                   clk ? clk_get_rate(clk) / 1000000 : freq);
2708
2709         return 0;
2710 }