drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

   1 /*
   2  * Copyright (c) 2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  */
  10
  11 #include "hisi_sas.h"
  12 #define DRV_NAME "hisi_sas_v3_hw"
  13
  14 /* global registers need init*/
  15 #define DLVRY_QUEUE_ENABLE              0x0
  16 #define IOST_BASE_ADDR_LO               0x8
  17 #define IOST_BASE_ADDR_HI               0xc
  18 #define ITCT_BASE_ADDR_LO               0x10
  19 #define ITCT_BASE_ADDR_HI               0x14
  20 #define IO_BROKEN_MSG_ADDR_LO           0x18
  21 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  22 #define PHY_CONTEXT                     0x20
  23 #define PHY_STATE                       0x24
  24 #define PHY_PORT_NUM_MA                 0x28
  25 #define PHY_CONN_RATE                   0x30
  26 #define ITCT_CLR                        0x44
  27 #define ITCT_CLR_EN_OFF                 16
  28 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  29 #define ITCT_DEV_OFF                    0
  30 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  31 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  32 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  33 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  34 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  35 #define CFG_MAX_TAG                     0x68
  36 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  37 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  38 #define HGC_GET_ITV_TIME                0x90
  39 #define DEVICE_MSG_WORK_MODE            0x94
  40 #define OPENA_WT_CONTI_TIME             0x9c
  41 #define I_T_NEXUS_LOSS_TIME             0xa0
  42 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  43 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  44 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  45 #define CFG_AGING_TIME                  0xbc
  46 #define HGC_DFX_CFG2                    0xc0
  47 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  48 #define CFG_SET_ABORTED_IPTT_OFF        0
  49 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  50 #define CFG_SET_ABORTED_EN_OFF  12
  51 #define CFG_ABT_SET_IPTT_DONE   0xd8
  52 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  53 #define HGC_IOMB_PROC1_STATUS   0x104
  54 #define CHNL_INT_STATUS                 0x148
  55 #define HGC_AXI_FIFO_ERR_INFO  0x154
  56 #define AXI_ERR_INFO_OFF               0
  57 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  58 #define FIFO_ERR_INFO_OFF              8
  59 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  60 #define INT_COAL_EN                     0x19c
  61 #define OQ_INT_COAL_TIME                0x1a0
  62 #define OQ_INT_COAL_CNT                 0x1a4
  63 #define ENT_INT_COAL_TIME               0x1a8
  64 #define ENT_INT_COAL_CNT                0x1ac
  65 #define OQ_INT_SRC                      0x1b0
  66 #define OQ_INT_SRC_MSK                  0x1b4
  67 #define ENT_INT_SRC1                    0x1b8
  68 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  69 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  70 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  71 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
  72 #define ENT_INT_SRC2                    0x1bc
  73 #define ENT_INT_SRC3                    0x1c0
  74 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
  75 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
  76 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
  77 #define ENT_INT_SRC3_AXI_OFF                    11
  78 #define ENT_INT_SRC3_FIFO_OFF                   12
  79 #define ENT_INT_SRC3_LM_OFF                             14
  80 #define ENT_INT_SRC3_ITC_INT_OFF        15
  81 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
  82 #define ENT_INT_SRC3_ABT_OFF            16
  83 #define ENT_INT_SRC_MSK1                0x1c4
  84 #define ENT_INT_SRC_MSK2                0x1c8
  85 #define ENT_INT_SRC_MSK3                0x1cc
  86 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
  87 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
  88 #define CHNL_ENT_INT_MSK                        0x1d4
  89 #define HGC_COM_INT_MSK                         0x1d8
  90 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
  91 #define SAS_ECC_INTR                    0x1e8
  92 #define SAS_ECC_INTR_MSK                0x1ec
  93 #define HGC_ERR_STAT_EN                 0x238
  94 #define CQE_SEND_CNT                    0x248
  95 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
  96 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
  97 #define DLVRY_Q_0_DEPTH                 0x268
  98 #define DLVRY_Q_0_WR_PTR                0x26c
  99 #define DLVRY_Q_0_RD_PTR                0x270
 100 #define HYPER_STREAM_ID_EN_CFG          0xc80
 101 #define OQ0_INT_SRC_MSK                 0xc90
 102 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 103 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 104 #define COMPL_Q_0_DEPTH                 0x4e8
 105 #define COMPL_Q_0_WR_PTR                0x4ec
 106 #define COMPL_Q_0_RD_PTR                0x4f0
 107 #define AWQOS_AWCACHE_CFG       0xc84
 108 #define ARQOS_ARCACHE_CFG       0xc88
 109 #define HILINK_ERR_DFX          0xe04
 110 #define SAS_GPIO_CFG_0          0x1000
 111 #define SAS_GPIO_CFG_1          0x1004
 112 #define SAS_GPIO_TX_0_1 0x1040
 113 #define SAS_CFG_DRIVE_VLD       0x1070
 114
 115 /* phy registers requiring init */
 116 #define PORT_BASE                       (0x2000)
 117 #define PHY_CFG                         (PORT_BASE + 0x0)
 118 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 119 #define PHY_CFG_ENA_OFF                 0
 120 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 121 #define PHY_CFG_DC_OPT_OFF              2
 122 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 123 #define PHY_CFG_PHY_RST_OFF             3
 124 #define PHY_CFG_PHY_RST_MSK             (0x1 << PHY_CFG_PHY_RST_OFF)
 125 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 126 #define PHY_CTRL                        (PORT_BASE + 0x14)
 127 #define PHY_CTRL_RESET_OFF              0
 128 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 129 #define SL_CFG                          (PORT_BASE + 0x84)
 130 #define SL_CONTROL                      (PORT_BASE + 0x94)
 131 #define SL_CONTROL_NOTIFY_EN_OFF        0
 132 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 133 #define SL_CTA_OFF              17
 134 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 135 #define RX_PRIMS_STATUS                 (PORT_BASE + 0x98)
 136 #define RX_BCAST_CHG_OFF                1
 137 #define RX_BCAST_CHG_MSK                (0x1 << RX_BCAST_CHG_OFF)
 138 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 139 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 140 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 141 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 142 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 143 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 144 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 145 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 146 #define CT3_OFF         1
 147 #define CT3_MSK         (0x1 << CT3_OFF)
 148 #define TX_HARDRST_OFF          2
 149 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 150 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 151 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 152 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 153 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 154 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 155 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 156 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 157 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 158 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 159 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 160 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 161 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 162 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 163 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 164 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 165 #define CHL_INT0_NOT_RDY_OFF            4
 166 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 167 #define CHL_INT0_PHY_RDY_OFF            5
 168 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 169 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 170 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF    15
 171 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
 172 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF    17
 173 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
 174 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 175 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 176 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 177 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 178 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 179 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 180 #define CHL_INT2_RX_INVLD_DW_OFF        30
 181 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 182 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 183 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 184 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 185 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 186 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 187 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 188 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 189 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 190 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 191 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 192 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 193 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 194 #define DMA_TX_STATUS_BUSY_OFF          0
 195 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 196 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 197 #define DMA_RX_STATUS_BUSY_OFF          0
 198 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 199
 200 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 201 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 202 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 203 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 204 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 205
 206 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 207 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 208 #error Max ITCT exceeded
 209 #endif
 210
 211 #define AXI_MASTER_CFG_BASE             (0x5000)
 212 #define AM_CTRL_GLOBAL                  (0x0)
 213 #define AM_CTRL_SHUTDOWN_REQ_OFF        0
 214 #define AM_CTRL_SHUTDOWN_REQ_MSK        (0x1 << AM_CTRL_SHUTDOWN_REQ_OFF)
 215 #define AM_CURR_TRANS_RETURN    (0x150)
 216
 217 #define AM_CFG_MAX_TRANS                (0x5010)
 218 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 219 #define AXI_CFG                                 (0x5100)
 220 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 221 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF  0
 222 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
 223 #define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF  8
 224 #define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 225
 226 /* RAS registers need init */
 227 #define RAS_BASE                (0x6000)
 228 #define SAS_RAS_INTR0                   (RAS_BASE)
 229 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 230 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 231 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 232 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 233 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 234 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 235
 236 /* HW dma structures */
 237 /* Delivery queue header */
 238 /* dw0 */
 239 #define CMD_HDR_ABORT_FLAG_OFF          0
 240 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 241 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 242 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 243 #define CMD_HDR_RESP_REPORT_OFF         5
 244 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 245 #define CMD_HDR_TLR_CTRL_OFF            6
 246 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 247 #define CMD_HDR_PORT_OFF                18
 248 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 249 #define CMD_HDR_PRIORITY_OFF            27
 250 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 251 #define CMD_HDR_CMD_OFF                 29
 252 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 253 /* dw1 */
 254 #define CMD_HDR_UNCON_CMD_OFF   3
 255 #define CMD_HDR_DIR_OFF                 5
 256 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 257 #define CMD_HDR_RESET_OFF               7
 258 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 259 #define CMD_HDR_VDTL_OFF                10
 260 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 261 #define CMD_HDR_FRAME_TYPE_OFF          11
 262 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 263 #define CMD_HDR_DEV_ID_OFF              16
 264 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 265 /* dw2 */
 266 #define CMD_HDR_CFL_OFF                 0
 267 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 268 #define CMD_HDR_NCQ_TAG_OFF             10
 269 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 270 #define CMD_HDR_MRFL_OFF                15
 271 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 272 #define CMD_HDR_SG_MOD_OFF              24
 273 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 274 /* dw3 */
 275 #define CMD_HDR_IPTT_OFF                0
 276 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 277 /* dw6 */
 278 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 279 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 280 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 281 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 282 /* dw7 */
 283 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 284 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 285 #define CMD_HDR_ABORT_IPTT_OFF          16
 286 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 287
 288 /* Completion header */
 289 /* dw0 */
 290 #define CMPLT_HDR_CMPLT_OFF             0
 291 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 292 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 293 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 294 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 295 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 296 #define CMPLT_HDR_ERX_OFF               12
 297 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 298 #define CMPLT_HDR_ABORT_STAT_OFF        13
 299 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 300 /* abort_stat */
 301 #define STAT_IO_NOT_VALID               0x1
 302 #define STAT_IO_NO_DEVICE               0x2
 303 #define STAT_IO_COMPLETE                0x3
 304 #define STAT_IO_ABORTED                 0x4
 305 /* dw1 */
 306 #define CMPLT_HDR_IPTT_OFF              0
 307 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 308 #define CMPLT_HDR_DEV_ID_OFF            16
 309 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 310 /* dw3 */
 311 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 312 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 313
 314 /* ITCT header */
 315 /* qw0 */
 316 #define ITCT_HDR_DEV_TYPE_OFF           0
 317 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 318 #define ITCT_HDR_VALID_OFF              2
 319 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 320 #define ITCT_HDR_MCR_OFF                5
 321 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 322 #define ITCT_HDR_VLN_OFF                9
 323 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 324 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 325 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 326 #define ITCT_HDR_PORT_ID_OFF            28
 327 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 328 /* qw2 */
 329 #define ITCT_HDR_INLT_OFF               0
 330 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 331 #define ITCT_HDR_RTOLT_OFF              48
 332 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 333
 334 struct hisi_sas_complete_v3_hdr {
 335         __le32 dw0;
 336         __le32 dw1;
 337         __le32 act;
 338         __le32 dw3;
 339 };
 340
 341 struct hisi_sas_err_record_v3 {
 342         /* dw0 */
 343         __le32 trans_tx_fail_type;
 344
 345         /* dw1 */
 346         __le32 trans_rx_fail_type;
 347
 348         /* dw2 */
 349         __le16 dma_tx_err_type;
 350         __le16 sipc_rx_err_type;
 351
 352         /* dw3 */
 353         __le32 dma_rx_err_type;
 354 };
 355
 356 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 357 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 358
 359 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 360 #define HISI_SAS_MSI_COUNT_V3_HW 32
 361
 362 #define DIR_NO_DATA 0
 363 #define DIR_TO_INI 1
 364 #define DIR_TO_DEVICE 2
 365 #define DIR_RESERVED 3
 366
 367 #define FIS_CMD_IS_UNCONSTRAINED(fis) \
 368         ((fis.command == ATA_CMD_READ_LOG_EXT) || \
 369         (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
 370         ((fis.command == ATA_CMD_DEV_RESET) && \
 371         ((fis.control & ATA_SRST) != 0)))
 372
 373 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 374 {
 375         void __iomem *regs = hisi_hba->regs + off;
 376
 377         return readl(regs);
 378 }
 379
 380 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off)
 381 {
 382         void __iomem *regs = hisi_hba->regs + off;
 383
 384         return readl_relaxed(regs);
 385 }
 386
 387 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 388 {
 389         void __iomem *regs = hisi_hba->regs + off;
 390
 391         writel(val, regs);
 392 }
 393
 394 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 395                                  u32 off, u32 val)
 396 {
 397         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 398
 399         writel(val, regs);
 400 }
 401
 402 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 403                                       int phy_no, u32 off)
 404 {
 405         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 406
 407         return readl(regs);
 408 }
 409
 410 #define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,          \
 411                                      timeout_us)                        \
 412 ({                                                                      \
 413         void __iomem *regs = hisi_hba->regs + off;                      \
 414         readl_poll_timeout(regs, val, cond, delay_us, timeout_us);      \
 415 })
 416
 417 #define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,   \
 418                                             timeout_us)                 \
 419 ({                                                                      \
 420         void __iomem *regs = hisi_hba->regs + off;                      \
 421         readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
 422 })
 423
 424 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 425 {
 426         struct pci_dev *pdev = hisi_hba->pci_dev;
 427         int i;
 428
 429         /* Global registers init */
 430         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 431                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 432         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 433         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 434         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 435         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 436         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 437         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 438         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 439         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 440         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 441         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 442         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 443         if (pdev->revision >= 0x21)
 444                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
 445         else
 446                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 447         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 448         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 449         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 450         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 451         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 452         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 453         for (i = 0; i < hisi_hba->queue_count; i++)
 454                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 455
 456         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 457
 458         for (i = 0; i < hisi_hba->n_phy; i++) {
 459                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 460                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 461                 u32 prog_phy_link_rate = 0x800;
 462
 463                 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
 464                                 SAS_LINK_RATE_1_5_GBPS)) {
 465                         prog_phy_link_rate = 0x855;
 466                 } else {
 467                         enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
 468
 469                         prog_phy_link_rate =
 470                                 hisi_sas_get_prog_phy_linkrate_mask(max) |
 471                                 0x800;
 472                 }
 473                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
 474                         prog_phy_link_rate);
 475                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 476                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 477                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 478                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 479                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 480                 if (pdev->revision >= 0x21)
 481                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 482                                         0xffffffff);
 483                 else
 484                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 485                                         0xff87ffff);
 486                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 487                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 488                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 489                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 490                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 491                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 492                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 493                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 494                 hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01);
 495                 hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32);
 496                 /* used for 12G negotiate */
 497                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 498         }
 499
 500         for (i = 0; i < hisi_hba->queue_count; i++) {
 501                 /* Delivery queue */
 502                 hisi_sas_write32(hisi_hba,
 503                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 504                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 505
 506                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 507                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 508
 509                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 510                                  HISI_SAS_QUEUE_SLOTS);
 511
 512                 /* Completion queue */
 513                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 514                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 515
 516                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 517                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 518
 519                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 520                                  HISI_SAS_QUEUE_SLOTS);
 521         }
 522
 523         /* itct */
 524         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 525                          lower_32_bits(hisi_hba->itct_dma));
 526
 527         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 528                          upper_32_bits(hisi_hba->itct_dma));
 529
 530         /* iost */
 531         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 532                          lower_32_bits(hisi_hba->iost_dma));
 533
 534         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 535                          upper_32_bits(hisi_hba->iost_dma));
 536
 537         /* breakpoint */
 538         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 539                          lower_32_bits(hisi_hba->breakpoint_dma));
 540
 541         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 542                          upper_32_bits(hisi_hba->breakpoint_dma));
 543
 544         /* SATA broken msg */
 545         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 546                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 547
 548         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 549                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 550
 551         /* SATA initial fis */
 552         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 553                          lower_32_bits(hisi_hba->initial_fis_dma));
 554
 555         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 556                          upper_32_bits(hisi_hba->initial_fis_dma));
 557
 558         /* RAS registers init */
 559         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 560         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 561         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 562         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 563
 564         /* LED registers init */
 565         hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
 566         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
 567         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
 568         /* Configure blink generator rate A to 1Hz and B to 4Hz */
 569         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
 570         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
 571 }
 572
 573 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 574 {
 575         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 576
 577         cfg &= ~PHY_CFG_DC_OPT_MSK;
 578         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 579         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 580 }
 581
 582 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 583 {
 584         struct sas_identify_frame identify_frame;
 585         u32 *identify_buffer;
 586
 587         memset(&identify_frame, 0, sizeof(identify_frame));
 588         identify_frame.dev_type = SAS_END_DEVICE;
 589         identify_frame.frame_type = 0;
 590         identify_frame._un1 = 1;
 591         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 592         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 593         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 594         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 595         identify_frame.phy_id = phy_no;
 596         identify_buffer = (u32 *)(&identify_frame);
 597
 598         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 599                         __swab32(identify_buffer[0]));
 600         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 601                         __swab32(identify_buffer[1]));
 602         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 603                         __swab32(identify_buffer[2]));
 604         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 605                         __swab32(identify_buffer[3]));
 606         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 607                         __swab32(identify_buffer[4]));
 608         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 609                         __swab32(identify_buffer[5]));
 610 }
 611
 612 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 613                              struct hisi_sas_device *sas_dev)
 614 {
 615         struct domain_device *device = sas_dev->sas_device;
 616         struct device *dev = hisi_hba->dev;
 617         u64 qw0, device_id = sas_dev->device_id;
 618         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 619         struct domain_device *parent_dev = device->parent;
 620         struct asd_sas_port *sas_port = device->port;
 621         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 622
 623         memset(itct, 0, sizeof(*itct));
 624
 625         /* qw0 */
 626         qw0 = 0;
 627         switch (sas_dev->dev_type) {
 628         case SAS_END_DEVICE:
 629         case SAS_EDGE_EXPANDER_DEVICE:
 630         case SAS_FANOUT_EXPANDER_DEVICE:
 631                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 632                 break;
 633         case SAS_SATA_DEV:
 634         case SAS_SATA_PENDING:
 635                 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 636                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 637                 else
 638                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 639                 break;
 640         default:
 641                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 642                          sas_dev->dev_type);
 643         }
 644
 645         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 646                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 647                 (1 << ITCT_HDR_VLN_OFF) |
 648                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 649                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 650                 (port->id << ITCT_HDR_PORT_ID_OFF));
 651         itct->qw0 = cpu_to_le64(qw0);
 652
 653         /* qw1 */
 654         memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 655         itct->sas_addr = __swab64(itct->sas_addr);
 656
 657         /* qw2 */
 658         if (!dev_is_sata(device))
 659                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 660                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 661 }
 662
 663 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 664                               struct hisi_sas_device *sas_dev)
 665 {
 666         DECLARE_COMPLETION_ONSTACK(completion);
 667         u64 dev_id = sas_dev->device_id;
 668         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 669         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 670
 671         sas_dev->completion = &completion;
 672
 673         /* clear the itct interrupt state */
 674         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 675                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 676                                  ENT_INT_SRC3_ITC_INT_MSK);
 677
 678         /* clear the itct table*/
 679         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 680         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 681
 682         wait_for_completion(sas_dev->completion);
 683         memset(itct, 0, sizeof(struct hisi_sas_itct));
 684 }
 685
 686 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 687                                 struct domain_device *device)
 688 {
 689         struct hisi_sas_slot *slot, *slot2;
 690         struct hisi_sas_device *sas_dev = device->lldd_dev;
 691         u32 cfg_abt_set_query_iptt;
 692
 693         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 694                 CFG_ABT_SET_QUERY_IPTT);
 695         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 696                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 697                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 698                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 699                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 700                         cfg_abt_set_query_iptt);
 701         }
 702         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 703         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 704                 cfg_abt_set_query_iptt);
 705         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 706                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 707 }
 708
 709 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 710 {
 711         struct device *dev = hisi_hba->dev;
 712         int ret;
 713         u32 val;
 714
 715         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 716
 717         /* Disable all of the PHYs */
 718         hisi_sas_stop_phys(hisi_hba);
 719         udelay(50);
 720
 721         /* Ensure axi bus idle */
 722         ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
 723                                            20000, 1000000);
 724         if (ret) {
 725                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 726                 return -EIO;
 727         }
 728
 729         if (ACPI_HANDLE(dev)) {
 730                 acpi_status s;
 731
 732                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 733                 if (ACPI_FAILURE(s)) {
 734                         dev_err(dev, "Reset failed\n");
 735                         return -EIO;
 736                 }
 737         } else {
 738                 dev_err(dev, "no reset method!\n");
 739                 return -EINVAL;
 740         }
 741
 742         return 0;
 743 }
 744
 745 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 746 {
 747         struct device *dev = hisi_hba->dev;
 748         int rc;
 749
 750         rc = reset_hw_v3_hw(hisi_hba);
 751         if (rc) {
 752                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 753                 return rc;
 754         }
 755
 756         msleep(100);
 757         init_reg_v3_hw(hisi_hba);
 758
 759         return 0;
 760 }
 761
 762 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 763 {
 764         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 765
 766         cfg |= PHY_CFG_ENA_MSK;
 767         cfg &= ~PHY_CFG_PHY_RST_MSK;
 768         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 769 }
 770
 771 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 772 {
 773         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 774         u32 state;
 775
 776         cfg &= ~PHY_CFG_ENA_MSK;
 777         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 778
 779         mdelay(50);
 780
 781         state = hisi_sas_read32(hisi_hba, PHY_STATE);
 782         if (state & BIT(phy_no)) {
 783                 cfg |= PHY_CFG_PHY_RST_MSK;
 784                 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 785         }
 786 }
 787
 788 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 789 {
 790         config_id_frame_v3_hw(hisi_hba, phy_no);
 791         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 792         enable_phy_v3_hw(hisi_hba, phy_no);
 793 }
 794
 795 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 796 {
 797         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 798         u32 txid_auto;
 799
 800         disable_phy_v3_hw(hisi_hba, phy_no);
 801         if (phy->identify.device_type == SAS_END_DEVICE) {
 802                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 803                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 804                                         txid_auto | TX_HARDRST_MSK);
 805         }
 806         msleep(100);
 807         start_phy_v3_hw(hisi_hba, phy_no);
 808 }
 809
 810 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 811 {
 812         return SAS_LINK_RATE_12_0_GBPS;
 813 }
 814
 815 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 816 {
 817         int i;
 818
 819         for (i = 0; i < hisi_hba->n_phy; i++) {
 820                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 821                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 822
 823                 if (!sas_phy->phy->enabled)
 824                         continue;
 825
 826                 start_phy_v3_hw(hisi_hba, i);
 827         }
 828 }
 829
 830 static void sl_notify_ssp_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 831 {
 832         u32 sl_control;
 833
 834         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 835         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
 836         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 837         msleep(1);
 838         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 839         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
 840         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 841 }
 842
 843 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
 844 {
 845         int i, bitmap = 0;
 846         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
 847         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 848
 849         for (i = 0; i < hisi_hba->n_phy; i++)
 850                 if (phy_state & BIT(i))
 851                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
 852                                 bitmap |= BIT(i);
 853
 854         return bitmap;
 855 }
 856
 857 /**
 858  * The callpath to this function and upto writing the write
 859  * queue pointer should be safe from interruption.
 860  */
 861 static int
 862 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 863 {
 864         struct device *dev = hisi_hba->dev;
 865         int queue = dq->id;
 866         u32 r, w;
 867
 868         w = dq->wr_point;
 869         r = hisi_sas_read32_relaxed(hisi_hba,
 870                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
 871         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 872                 dev_warn(dev, "full queue=%d r=%d w=%d\n",
 873                                 queue, r, w);
 874                 return -EAGAIN;
 875         }
 876
 877         dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
 878
 879         return w;
 880 }
 881
 882 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 883 {
 884         struct hisi_hba *hisi_hba = dq->hisi_hba;
 885         struct hisi_sas_slot *s, *s1, *s2 = NULL;
 886         int dlvry_queue = dq->id;
 887         int wp;
 888
 889         list_for_each_entry_safe(s, s1, &dq->list, delivery) {
 890                 if (!s->ready)
 891                         break;
 892                 s2 = s;
 893                 list_del(&s->delivery);
 894         }
 895
 896         if (!s2)
 897                 return;
 898
 899         /*
 900          * Ensure that memories for slots built on other CPUs is observed.
 901          */
 902         smp_rmb();
 903         wp = (s2->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
 904
 905         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 906 }
 907
 908 static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 909                               struct hisi_sas_slot *slot,
 910                               struct hisi_sas_cmd_hdr *hdr,
 911                               struct scatterlist *scatter,
 912                               int n_elem)
 913 {
 914         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
 915         struct scatterlist *sg;
 916         int i;
 917
 918         for_each_sg(scatter, sg, n_elem, i) {
 919                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 920
 921                 entry->addr = cpu_to_le64(sg_dma_address(sg));
 922                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
 923                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
 924                 entry->data_off = 0;
 925         }
 926
 927         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 928
 929         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
 930 }
 931
 932 static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 933                           struct hisi_sas_slot *slot)
 934 {
 935         struct sas_task *task = slot->task;
 936         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 937         struct domain_device *device = task->dev;
 938         struct hisi_sas_device *sas_dev = device->lldd_dev;
 939         struct hisi_sas_port *port = slot->port;
 940         struct sas_ssp_task *ssp_task = &task->ssp_task;
 941         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 942         struct hisi_sas_tmf_task *tmf = slot->tmf;
 943         int has_data = 0, priority = !!tmf;
 944         u8 *buf_cmd;
 945         u32 dw1 = 0, dw2 = 0;
 946
 947         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
 948                                (2 << CMD_HDR_TLR_CTRL_OFF) |
 949                                (port->id << CMD_HDR_PORT_OFF) |
 950                                (priority << CMD_HDR_PRIORITY_OFF) |
 951                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 952
 953         dw1 = 1 << CMD_HDR_VDTL_OFF;
 954         if (tmf) {
 955                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 956                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 957         } else {
 958                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
 959                 switch (scsi_cmnd->sc_data_direction) {
 960                 case DMA_TO_DEVICE:
 961                         has_data = 1;
 962                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
 963                         break;
 964                 case DMA_FROM_DEVICE:
 965                         has_data = 1;
 966                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
 967                         break;
 968                 default:
 969                         dw1 &= ~CMD_HDR_DIR_MSK;
 970                 }
 971         }
 972
 973         /* map itct entry */
 974         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 975         hdr->dw1 = cpu_to_le32(dw1);
 976
 977         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
 978               + 3) / 4) << CMD_HDR_CFL_OFF) |
 979               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
 980               (2 << CMD_HDR_SG_MOD_OFF);
 981         hdr->dw2 = cpu_to_le32(dw2);
 982         hdr->transfer_tags = cpu_to_le32(slot->idx);
 983
 984         if (has_data)
 985                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 986                                         slot->n_elem);
 987
 988         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 989         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
 990         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 991
 992         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
 993                 sizeof(struct ssp_frame_hdr);
 994
 995         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
 996         if (!tmf) {
 997                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
 998                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 999         } else {
1000                 buf_cmd[10] = tmf->tmf;
1001                 switch (tmf->tmf) {
1002                 case TMF_ABORT_TASK:
1003                 case TMF_QUERY_TASK:
1004                         buf_cmd[12] =
1005                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
1006                         buf_cmd[13] =
1007                                 tmf->tag_of_task_to_be_managed & 0xff;
1008                         break;
1009                 default:
1010                         break;
1011                 }
1012         }
1013 }
1014
1015 static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
1016                           struct hisi_sas_slot *slot)
1017 {
1018         struct sas_task *task = slot->task;
1019         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1020         struct domain_device *device = task->dev;
1021         struct hisi_sas_port *port = slot->port;
1022         struct scatterlist *sg_req;
1023         struct hisi_sas_device *sas_dev = device->lldd_dev;
1024         dma_addr_t req_dma_addr;
1025         unsigned int req_len;
1026
1027         /* req */
1028         sg_req = &task->smp_task.smp_req;
1029         req_len = sg_dma_len(sg_req);
1030         req_dma_addr = sg_dma_address(sg_req);
1031
1032         /* create header */
1033         /* dw0 */
1034         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1035                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1036                                (2 << CMD_HDR_CMD_OFF)); /* smp */
1037
1038         /* map itct entry */
1039         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1040                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
1041                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1042
1043         /* dw2 */
1044         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1045                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1046                                CMD_HDR_MRFL_OFF));
1047
1048         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1049
1050         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1051         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1052
1053 }
1054
1055 static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1056                           struct hisi_sas_slot *slot)
1057 {
1058         struct sas_task *task = slot->task;
1059         struct domain_device *device = task->dev;
1060         struct domain_device *parent_dev = device->parent;
1061         struct hisi_sas_device *sas_dev = device->lldd_dev;
1062         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1063         struct asd_sas_port *sas_port = device->port;
1064         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1065         u8 *buf_cmd;
1066         int has_data = 0, hdr_tag = 0;
1067         u32 dw1 = 0, dw2 = 0;
1068
1069         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1070         if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
1071                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1072         else
1073                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
1074
1075         switch (task->data_dir) {
1076         case DMA_TO_DEVICE:
1077                 has_data = 1;
1078                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1079                 break;
1080         case DMA_FROM_DEVICE:
1081                 has_data = 1;
1082                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1083                 break;
1084         default:
1085                 dw1 &= ~CMD_HDR_DIR_MSK;
1086         }
1087
1088         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1089                         (task->ata_task.fis.control & ATA_SRST))
1090                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1091
1092         dw1 |= (hisi_sas_get_ata_protocol(
1093                 &task->ata_task.fis, task->data_dir))
1094                 << CMD_HDR_FRAME_TYPE_OFF;
1095         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1096
1097         if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1098                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1099
1100         hdr->dw1 = cpu_to_le32(dw1);
1101
1102         /* dw2 */
1103         if (task->ata_task.use_ncq && hisi_sas_get_ncq_tag(task, &hdr_tag)) {
1104                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1105                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1106         }
1107
1108         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1109                         2 << CMD_HDR_SG_MOD_OFF;
1110         hdr->dw2 = cpu_to_le32(dw2);
1111
1112         /* dw3 */
1113         hdr->transfer_tags = cpu_to_le32(slot->idx);
1114
1115         if (has_data)
1116                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1117                                         slot->n_elem);
1118
1119         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1120         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1121         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1122
1123         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1124
1125         if (likely(!task->ata_task.device_control_reg_update))
1126                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1127         /* fill in command FIS */
1128         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1129 }
1130
1131 static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1132                 struct hisi_sas_slot *slot,
1133                 int device_id, int abort_flag, int tag_to_abort)
1134 {
1135         struct sas_task *task = slot->task;
1136         struct domain_device *dev = task->dev;
1137         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1138         struct hisi_sas_port *port = slot->port;
1139
1140         /* dw0 */
1141         hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
1142                                (port->id << CMD_HDR_PORT_OFF) |
1143                                    (dev_is_sata(dev)
1144                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1145                                         (abort_flag
1146                                          << CMD_HDR_ABORT_FLAG_OFF));
1147
1148         /* dw1 */
1149         hdr->dw1 = cpu_to_le32(device_id
1150                         << CMD_HDR_DEV_ID_OFF);
1151
1152         /* dw7 */
1153         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1154         hdr->transfer_tags = cpu_to_le32(slot->idx);
1155
1156 }
1157
1158 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1159 {
1160         int i, res;
1161         u32 context, port_id, link_rate;
1162         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1163         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1164         struct device *dev = hisi_hba->dev;
1165         unsigned long flags;
1166
1167         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1168
1169         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1170         port_id = (port_id >> (4 * phy_no)) & 0xf;
1171         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1172         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1173
1174         if (port_id == 0xf) {
1175                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1176                 res = IRQ_NONE;
1177                 goto end;
1178         }
1179         sas_phy->linkrate = link_rate;
1180         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1181
1182         /* Check for SATA dev */
1183         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1184         if (context & (1 << phy_no)) {
1185                 struct hisi_sas_initial_fis *initial_fis;
1186                 struct dev_to_host_fis *fis;
1187                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1188
1189                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1190                 initial_fis = &hisi_hba->initial_fis[phy_no];
1191                 fis = &initial_fis->fis;
1192
1193                 /* check ERR bit of Status Register */
1194                 if (fis->status & ATA_ERR) {
1195                         dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n",
1196                                  phy_no, fis->status);
1197                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1198                         res = IRQ_NONE;
1199                         goto end;
1200                 }
1201
1202                 sas_phy->oob_mode = SATA_OOB_MODE;
1203                 attached_sas_addr[0] = 0x50;
1204                 attached_sas_addr[7] = phy_no;
1205                 memcpy(sas_phy->attached_sas_addr,
1206                        attached_sas_addr,
1207                        SAS_ADDR_SIZE);
1208                 memcpy(sas_phy->frame_rcvd, fis,
1209                        sizeof(struct dev_to_host_fis));
1210                 phy->phy_type |= PORT_TYPE_SATA;
1211                 phy->identify.device_type = SAS_SATA_DEV;
1212                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1213                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1214         } else {
1215                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1216                 struct sas_identify_frame *id =
1217                         (struct sas_identify_frame *)frame_rcvd;
1218
1219                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1220                 for (i = 0; i < 6; i++) {
1221                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1222                                                RX_IDAF_DWORD0 + (i * 4));
1223                         frame_rcvd[i] = __swab32(idaf);
1224                 }
1225                 sas_phy->oob_mode = SAS_OOB_MODE;
1226                 memcpy(sas_phy->attached_sas_addr,
1227                        &id->sas_addr,
1228                        SAS_ADDR_SIZE);
1229                 phy->phy_type |= PORT_TYPE_SAS;
1230                 phy->identify.device_type = id->dev_type;
1231                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1232                 if (phy->identify.device_type == SAS_END_DEVICE)
1233                         phy->identify.target_port_protocols =
1234                                 SAS_PROTOCOL_SSP;
1235                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1236                         phy->identify.target_port_protocols =
1237                                 SAS_PROTOCOL_SMP;
1238         }
1239
1240         phy->port_id = port_id;
1241         phy->phy_attached = 1;
1242         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1243         res = IRQ_HANDLED;
1244         spin_lock_irqsave(&phy->lock, flags);
1245         if (phy->reset_completion) {
1246                 phy->in_reset = 0;
1247                 complete(phy->reset_completion);
1248         }
1249         spin_unlock_irqrestore(&phy->lock, flags);
1250 end:
1251         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1252                              CHL_INT0_SL_PHY_ENABLE_MSK);
1253         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1254
1255         return res;
1256 }
1257
1258 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1259 {
1260         u32 phy_state, sl_ctrl, txid_auto;
1261         struct device *dev = hisi_hba->dev;
1262
1263         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1264
1265         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1266         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1267         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1268
1269         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1270         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1271                                                 sl_ctrl&(~SL_CTA_MSK));
1272
1273         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1274         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1275                                                 txid_auto | CT3_MSK);
1276
1277         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1278         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1279
1280         return IRQ_HANDLED;
1281 }
1282
1283 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1284 {
1285         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1286         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1287         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1288         u32 bcast_status;
1289
1290         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1291         bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
1292         if ((bcast_status & RX_BCAST_CHG_MSK) &&
1293             !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
1294                 sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1295         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1296                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1297         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1298
1299         return IRQ_HANDLED;
1300 }
1301
1302 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1303 {
1304         struct hisi_hba *hisi_hba = p;
1305         u32 irq_msk;
1306         int phy_no = 0;
1307         irqreturn_t res = IRQ_NONE;
1308
1309         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1310                                 & 0x11111111;
1311         while (irq_msk) {
1312                 if (irq_msk  & 1) {
1313                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1314                                                             CHL_INT0);
1315                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1316                         int rdy = phy_state & (1 << phy_no);
1317
1318                         if (rdy) {
1319                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1320                                         /* phy up */
1321                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1322                                                         == IRQ_HANDLED)
1323                                                 res = IRQ_HANDLED;
1324                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1325                                         /* phy bcast */
1326                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1327                                                         == IRQ_HANDLED)
1328                                                 res = IRQ_HANDLED;
1329                         } else {
1330                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1331                                         /* phy down */
1332                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1333                                                         == IRQ_HANDLED)
1334                                                 res = IRQ_HANDLED;
1335                         }
1336                 }
1337                 irq_msk >>= 4;
1338                 phy_no++;
1339         }
1340
1341         return res;
1342 }
1343
1344 static const struct hisi_sas_hw_error port_axi_error[] = {
1345         {
1346                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1347                 .msg = "dma_tx_axi_wr_err",
1348         },
1349         {
1350                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1351                 .msg = "dma_tx_axi_rd_err",
1352         },
1353         {
1354                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1355                 .msg = "dma_rx_axi_wr_err",
1356         },
1357         {
1358                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1359                 .msg = "dma_rx_axi_rd_err",
1360         },
1361 };
1362
1363 static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1364 {
1365         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1);
1366         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1_MSK);
1367         struct device *dev = hisi_hba->dev;
1368         int i;
1369
1370         irq_value &= ~irq_msk;
1371         if (!irq_value)
1372                 return;
1373
1374         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1375                 const struct hisi_sas_hw_error *error = &port_axi_error[i];
1376
1377                 if (!(irq_value & error->irq_msk))
1378                         continue;
1379
1380                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1381                         error->msg, phy_no, irq_value);
1382                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1383         }
1384
1385         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value);
1386 }
1387
1388 static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1389 {
1390         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1391         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
1392         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1393         struct pci_dev *pci_dev = hisi_hba->pci_dev;
1394         struct device *dev = hisi_hba->dev;
1395
1396         irq_value &= ~irq_msk;
1397         if (!irq_value)
1398                 return;
1399
1400         if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1401                 dev_warn(dev, "phy%d identify timeout\n", phy_no);
1402                 hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1403         }
1404
1405         if (irq_value & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1406                 u32 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1407                                 STP_LINK_TIMEOUT_STATE);
1408
1409                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1410                          phy_no, reg_value);
1411                 if (reg_value & BIT(4))
1412                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1413         }
1414
1415         if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1416             (pci_dev->revision == 0x20)) {
1417                 u32 reg_value;
1418                 int rc;
1419
1420                 rc = hisi_sas_read32_poll_timeout_atomic(
1421                                 HILINK_ERR_DFX, reg_value,
1422                                 !((reg_value >> 8) & BIT(phy_no)),
1423                                 1000, 10000);
1424                 if (rc)
1425                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1426         }
1427
1428         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, irq_value);
1429 }
1430
1431 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1432 {
1433         struct hisi_hba *hisi_hba = p;
1434         u32 irq_msk;
1435         int phy_no = 0;
1436
1437         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1438                                 & 0xeeeeeeee;
1439
1440         while (irq_msk) {
1441                 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no,
1442                                                      CHL_INT0);
1443
1444                 if (irq_msk & (4 << (phy_no * 4)))
1445                         handle_chl_int1_v3_hw(hisi_hba, phy_no);
1446
1447                 if (irq_msk & (8 << (phy_no * 4)))
1448                         handle_chl_int2_v3_hw(hisi_hba, phy_no);
1449
1450                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
1451                         hisi_sas_phy_write32(hisi_hba, phy_no,
1452                                         CHL_INT0, irq_value0
1453                                         & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1454                                         & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1455                                         & (~CHL_INT0_NOT_RDY_MSK));
1456                 }
1457                 irq_msk &= ~(0xe << (phy_no * 4));
1458                 phy_no++;
1459         }
1460
1461         return IRQ_HANDLED;
1462 }
1463
1464 static const struct hisi_sas_hw_error axi_error[] = {
1465         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1466         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1467         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1468         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1469         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1470         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1471         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1472         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1473         {},
1474 };
1475
1476 static const struct hisi_sas_hw_error fifo_error[] = {
1477         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1478         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1479         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1480         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1481         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1482         {},
1483 };
1484
1485 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1486         {
1487                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1488                 .msg = "write pointer and depth",
1489         },
1490         {
1491                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1492                 .msg = "iptt no match slot",
1493         },
1494         {
1495                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1496                 .msg = "read pointer and depth",
1497         },
1498         {
1499                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1500                 .reg = HGC_AXI_FIFO_ERR_INFO,
1501                 .sub = axi_error,
1502         },
1503         {
1504                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1505                 .reg = HGC_AXI_FIFO_ERR_INFO,
1506                 .sub = fifo_error,
1507         },
1508         {
1509                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1510                 .msg = "LM add/fetch list",
1511         },
1512         {
1513                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1514                 .msg = "SAS_HGC_ABT fetch LM list",
1515         },
1516 };
1517
1518 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
1519 {
1520         u32 irq_value, irq_msk;
1521         struct hisi_hba *hisi_hba = p;
1522         struct device *dev = hisi_hba->dev;
1523         struct pci_dev *pdev = hisi_hba->pci_dev;
1524         int i;
1525
1526         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1527         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
1528
1529         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
1530         irq_value &= ~irq_msk;
1531
1532         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
1533                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
1534
1535                 if (!(irq_value & error->irq_msk))
1536                         continue;
1537
1538                 if (error->sub) {
1539                         const struct hisi_sas_hw_error *sub = error->sub;
1540                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
1541
1542                         for (; sub->msk || sub->msg; sub++) {
1543                                 if (!(err_value & sub->msk))
1544                                         continue;
1545
1546                                 dev_err(dev, "%s error (0x%x) found!\n",
1547                                         sub->msg, irq_value);
1548                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1549                         }
1550                 } else {
1551                         dev_err(dev, "%s error (0x%x) found!\n",
1552                                 error->msg, irq_value);
1553                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1554                 }
1555
1556                 if (pdev->revision < 0x21) {
1557                         u32 reg_val;
1558
1559                         reg_val = hisi_sas_read32(hisi_hba,
1560                                                   AXI_MASTER_CFG_BASE +
1561                                                   AM_CTRL_GLOBAL);
1562                         reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
1563                         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
1564                                          AM_CTRL_GLOBAL, reg_val);
1565                 }
1566         }
1567
1568         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
1569                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
1570                 u32 dev_id = reg_val & ITCT_DEV_MSK;
1571                 struct hisi_sas_device *sas_dev =
1572                                 &hisi_hba->devices[dev_id];
1573
1574                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
1575                 dev_dbg(dev, "clear ITCT ok\n");
1576                 complete(sas_dev->completion);
1577         }
1578
1579         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
1580         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
1581
1582         return IRQ_HANDLED;
1583 }
1584
1585 static void
1586 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
1587                struct hisi_sas_slot *slot)
1588 {
1589         struct task_status_struct *ts = &task->task_status;
1590         struct hisi_sas_complete_v3_hdr *complete_queue =
1591                         hisi_hba->complete_hdr[slot->cmplt_queue];
1592         struct hisi_sas_complete_v3_hdr *complete_hdr =
1593                         &complete_queue[slot->cmplt_queue_slot];
1594         struct hisi_sas_err_record_v3 *record =
1595                         hisi_sas_status_buf_addr_mem(slot);
1596         u32 dma_rx_err_type = record->dma_rx_err_type;
1597         u32 trans_tx_fail_type = record->trans_tx_fail_type;
1598
1599         switch (task->task_proto) {
1600         case SAS_PROTOCOL_SSP:
1601                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1602                         ts->residual = trans_tx_fail_type;
1603                         ts->stat = SAS_DATA_UNDERRUN;
1604                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1605                         ts->stat = SAS_QUEUE_FULL;
1606                         slot->abort = 1;
1607                 } else {
1608                         ts->stat = SAS_OPEN_REJECT;
1609                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1610                 }
1611                 break;
1612         case SAS_PROTOCOL_SATA:
1613         case SAS_PROTOCOL_STP:
1614         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1615                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1616                         ts->residual = trans_tx_fail_type;
1617                         ts->stat = SAS_DATA_UNDERRUN;
1618                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1619                         ts->stat = SAS_PHY_DOWN;
1620                         slot->abort = 1;
1621                 } else {
1622                         ts->stat = SAS_OPEN_REJECT;
1623                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1624                 }
1625                 hisi_sas_sata_done(task, slot);
1626                 break;
1627         case SAS_PROTOCOL_SMP:
1628                 ts->stat = SAM_STAT_CHECK_CONDITION;
1629                 break;
1630         default:
1631                 break;
1632         }
1633 }
1634
1635 static int
1636 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
1637 {
1638         struct sas_task *task = slot->task;
1639         struct hisi_sas_device *sas_dev;
1640         struct device *dev = hisi_hba->dev;
1641         struct task_status_struct *ts;
1642         struct domain_device *device;
1643         struct sas_ha_struct *ha;
1644         enum exec_status sts;
1645         struct hisi_sas_complete_v3_hdr *complete_queue =
1646                         hisi_hba->complete_hdr[slot->cmplt_queue];
1647         struct hisi_sas_complete_v3_hdr *complete_hdr =
1648                         &complete_queue[slot->cmplt_queue_slot];
1649         unsigned long flags;
1650         bool is_internal = slot->is_internal;
1651
1652         if (unlikely(!task || !task->lldd_task || !task->dev))
1653                 return -EINVAL;
1654
1655         ts = &task->task_status;
1656         device = task->dev;
1657         ha = device->port->ha;
1658         sas_dev = device->lldd_dev;
1659
1660         spin_lock_irqsave(&task->task_state_lock, flags);
1661         task->task_state_flags &=
1662                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1663         spin_unlock_irqrestore(&task->task_state_lock, flags);
1664
1665         memset(ts, 0, sizeof(*ts));
1666         ts->resp = SAS_TASK_COMPLETE;
1667
1668         if (unlikely(!sas_dev)) {
1669                 dev_dbg(dev, "slot complete: port has not device\n");
1670                 ts->stat = SAS_PHY_DOWN;
1671                 goto out;
1672         }
1673
1674         /*
1675          * Use SAS+TMF status codes
1676          */
1677         switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
1678                         >> CMPLT_HDR_ABORT_STAT_OFF) {
1679         case STAT_IO_ABORTED:
1680                 /* this IO has been aborted by abort command */
1681                 ts->stat = SAS_ABORTED_TASK;
1682                 goto out;
1683         case STAT_IO_COMPLETE:
1684                 /* internal abort command complete */
1685                 ts->stat = TMF_RESP_FUNC_SUCC;
1686                 goto out;
1687         case STAT_IO_NO_DEVICE:
1688                 ts->stat = TMF_RESP_FUNC_COMPLETE;
1689                 goto out;
1690         case STAT_IO_NOT_VALID:
1691                 /*
1692                  * abort single IO, the controller can't find the IO
1693                  */
1694                 ts->stat = TMF_RESP_FUNC_FAILED;
1695                 goto out;
1696         default:
1697                 break;
1698         }
1699
1700         /* check for erroneous completion */
1701         if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
1702                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
1703
1704                 slot_err_v3_hw(hisi_hba, task, slot);
1705                 if (ts->stat != SAS_DATA_UNDERRUN)
1706                         dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
1707                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
1708                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
1709                                 slot->idx, task, sas_dev->device_id,
1710                                 complete_hdr->dw0, complete_hdr->dw1,
1711                                 complete_hdr->act, complete_hdr->dw3,
1712                                 error_info[0], error_info[1],
1713                                 error_info[2], error_info[3]);
1714                 if (unlikely(slot->abort))
1715                         return ts->stat;
1716                 goto out;
1717         }
1718
1719         switch (task->task_proto) {
1720         case SAS_PROTOCOL_SSP: {
1721                 struct ssp_response_iu *iu =
1722                         hisi_sas_status_buf_addr_mem(slot) +
1723                         sizeof(struct hisi_sas_err_record);
1724
1725                 sas_ssp_task_response(dev, task, iu);
1726                 break;
1727         }
1728         case SAS_PROTOCOL_SMP: {
1729                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
1730                 void *to;
1731
1732                 ts->stat = SAM_STAT_GOOD;
1733                 to = kmap_atomic(sg_page(sg_resp));
1734
1735                 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1,
1736                              DMA_FROM_DEVICE);
1737                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
1738                              DMA_TO_DEVICE);
1739                 memcpy(to + sg_resp->offset,
1740                         hisi_sas_status_buf_addr_mem(slot) +
1741                        sizeof(struct hisi_sas_err_record),
1742                        sg_dma_len(sg_resp));
1743                 kunmap_atomic(to);
1744                 break;
1745         }
1746         case SAS_PROTOCOL_SATA:
1747         case SAS_PROTOCOL_STP:
1748         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1749                 ts->stat = SAM_STAT_GOOD;
1750                 hisi_sas_sata_done(task, slot);
1751                 break;
1752         default:
1753                 ts->stat = SAM_STAT_CHECK_CONDITION;
1754                 break;
1755         }
1756
1757         if (!slot->port->port_attached) {
1758                 dev_warn(dev, "slot complete: port %d has removed\n",
1759                         slot->port->sas_port.id);
1760                 ts->stat = SAS_PHY_DOWN;
1761         }
1762
1763 out:
1764         sts = ts->stat;
1765         spin_lock_irqsave(&task->task_state_lock, flags);
1766         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
1767                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1768                 dev_info(dev, "slot complete: task(%p) aborted\n", task);
1769                 return SAS_ABORTED_TASK;
1770         }
1771         task->task_state_flags |= SAS_TASK_STATE_DONE;
1772         spin_unlock_irqrestore(&task->task_state_lock, flags);
1773         hisi_sas_slot_task_free(hisi_hba, task, slot);
1774
1775         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
1776                 spin_lock_irqsave(&device->done_lock, flags);
1777                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
1778                         spin_unlock_irqrestore(&device->done_lock, flags);
1779                         dev_info(dev, "slot complete: task(%p) ignored\n ",
1780                                  task);
1781                         return sts;
1782                 }
1783                 spin_unlock_irqrestore(&device->done_lock, flags);
1784         }
1785
1786         if (task->task_done)
1787                 task->task_done(task);
1788
1789         return sts;
1790 }
1791
1792 static void cq_tasklet_v3_hw(unsigned long val)
1793 {
1794         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
1795         struct hisi_hba *hisi_hba = cq->hisi_hba;
1796         struct hisi_sas_slot *slot;
1797         struct hisi_sas_complete_v3_hdr *complete_queue;
1798         u32 rd_point = cq->rd_point, wr_point;
1799         int queue = cq->id;
1800
1801         complete_queue = hisi_hba->complete_hdr[queue];
1802
1803         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
1804                                    (0x14 * queue));
1805
1806         while (rd_point != wr_point) {
1807                 struct hisi_sas_complete_v3_hdr *complete_hdr;
1808                 struct device *dev = hisi_hba->dev;
1809                 int iptt;
1810
1811                 complete_hdr = &complete_queue[rd_point];
1812
1813                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
1814                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
1815                         slot = &hisi_hba->slot_info[iptt];
1816                         slot->cmplt_queue_slot = rd_point;
1817                         slot->cmplt_queue = queue;
1818                         slot_complete_v3_hw(hisi_hba, slot);
1819                 } else
1820                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
1821
1822                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
1823                         rd_point = 0;
1824         }
1825
1826         /* update rd_point */
1827         cq->rd_point = rd_point;
1828         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
1829 }
1830
1831 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
1832 {
1833         struct hisi_sas_cq *cq = p;
1834         struct hisi_hba *hisi_hba = cq->hisi_hba;
1835         int queue = cq->id;
1836
1837         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
1838
1839         tasklet_schedule(&cq->tasklet);
1840
1841         return IRQ_HANDLED;
1842 }
1843
1844 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
1845 {
1846         struct device *dev = hisi_hba->dev;
1847         struct pci_dev *pdev = hisi_hba->pci_dev;
1848         int vectors, rc;
1849         int i, k;
1850         int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
1851
1852         vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1,
1853                                         max_msi, PCI_IRQ_MSI);
1854         if (vectors < max_msi) {
1855                 dev_err(dev, "could not allocate all msi (%d)\n", vectors);
1856                 return -ENOENT;
1857         }
1858
1859         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
1860                               int_phy_up_down_bcast_v3_hw, 0,
1861                               DRV_NAME " phy", hisi_hba);
1862         if (rc) {
1863                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
1864                 rc = -ENOENT;
1865                 goto free_irq_vectors;
1866         }
1867
1868         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
1869                               int_chnl_int_v3_hw, 0,
1870                               DRV_NAME " channel", hisi_hba);
1871         if (rc) {
1872                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
1873                 rc = -ENOENT;
1874                 goto free_phy_irq;
1875         }
1876
1877         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
1878                               fatal_axi_int_v3_hw, 0,
1879                               DRV_NAME " fatal", hisi_hba);
1880         if (rc) {
1881                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
1882                 rc = -ENOENT;
1883                 goto free_chnl_interrupt;
1884         }
1885
1886         /* Init tasklets for cq only */
1887         for (i = 0; i < hisi_hba->queue_count; i++) {
1888                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
1889                 struct tasklet_struct *t = &cq->tasklet;
1890
1891                 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16),
1892                                           cq_interrupt_v3_hw, 0,
1893                                           DRV_NAME " cq", cq);
1894                 if (rc) {
1895                         dev_err(dev,
1896                                 "could not request cq%d interrupt, rc=%d\n",
1897                                 i, rc);
1898                         rc = -ENOENT;
1899                         goto free_cq_irqs;
1900                 }
1901
1902                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
1903         }
1904
1905         return 0;
1906
1907 free_cq_irqs:
1908         for (k = 0; k < i; k++) {
1909                 struct hisi_sas_cq *cq = &hisi_hba->cq[k];
1910
1911                 free_irq(pci_irq_vector(pdev, k+16), cq);
1912         }
1913         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
1914 free_chnl_interrupt:
1915         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
1916 free_phy_irq:
1917         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
1918 free_irq_vectors:
1919         pci_free_irq_vectors(pdev);
1920         return rc;
1921 }
1922
1923 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
1924 {
1925         int rc;
1926
1927         rc = hw_init_v3_hw(hisi_hba);
1928         if (rc)
1929                 return rc;
1930
1931         rc = interrupt_init_v3_hw(hisi_hba);
1932         if (rc)
1933                 return rc;
1934
1935         return 0;
1936 }
1937
1938 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
1939                 struct sas_phy_linkrates *r)
1940 {
1941         enum sas_linkrate max = r->maximum_linkrate;
1942         u32 prog_phy_link_rate = 0x800;
1943
1944         prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
1945         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
1946                              prog_phy_link_rate);
1947 }
1948
1949 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
1950 {
1951         struct pci_dev *pdev = hisi_hba->pci_dev;
1952         int i;
1953
1954         synchronize_irq(pci_irq_vector(pdev, 1));
1955         synchronize_irq(pci_irq_vector(pdev, 2));
1956         synchronize_irq(pci_irq_vector(pdev, 11));
1957         for (i = 0; i < hisi_hba->queue_count; i++) {
1958                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
1959                 synchronize_irq(pci_irq_vector(pdev, i + 16));
1960         }
1961
1962         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
1963         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
1964         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
1965         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1966
1967         for (i = 0; i < hisi_hba->n_phy; i++) {
1968                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
1969                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
1970                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
1971                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
1972                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
1973         }
1974 }
1975
1976 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
1977 {
1978         return hisi_sas_read32(hisi_hba, PHY_STATE);
1979 }
1980
1981 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1982 {
1983         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1984         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1985         struct sas_phy *sphy = sas_phy->phy;
1986         u32 reg_value;
1987
1988         /* loss dword sync */
1989         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1990         sphy->loss_of_dword_sync_count += reg_value;
1991
1992         /* phy reset problem */
1993         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1994         sphy->phy_reset_problem_count += reg_value;
1995
1996         /* invalid dword */
1997         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1998         sphy->invalid_dword_count += reg_value;
1999
2000         /* disparity err */
2001         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
2002         sphy->running_disparity_error_count += reg_value;
2003
2004 }
2005
2006 static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
2007 {
2008         struct device *dev = hisi_hba->dev;
2009         u32 status, reg_val;
2010         int rc;
2011
2012         interrupt_disable_v3_hw(hisi_hba);
2013         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2014         hisi_sas_kill_tasklets(hisi_hba);
2015
2016         hisi_sas_stop_phys(hisi_hba);
2017
2018         mdelay(10);
2019
2020         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2021                                   AM_CTRL_GLOBAL);
2022         reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2023         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2024                          AM_CTRL_GLOBAL, reg_val);
2025
2026         /* wait until bus idle */
2027         rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2028                                           AM_CURR_TRANS_RETURN, status,
2029                                           status == 0x3, 10, 100);
2030         if (rc) {
2031                 dev_err(dev, "axi bus is not idle, rc=%d\n", rc);
2032                 return rc;
2033         }
2034
2035         return 0;
2036 }
2037
2038 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
2039 {
2040         struct device *dev = hisi_hba->dev;
2041         int rc;
2042
2043         rc = disable_host_v3_hw(hisi_hba);
2044         if (rc) {
2045                 dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
2046                 return rc;
2047         }
2048
2049         hisi_sas_init_mem(hisi_hba);
2050
2051         return hw_init_v3_hw(hisi_hba);
2052 }
2053
2054 static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
2055                         u8 reg_index, u8 reg_count, u8 *write_data)
2056 {
2057         struct device *dev = hisi_hba->dev;
2058         u32 *data = (u32 *)write_data;
2059         int i;
2060
2061         switch (reg_type) {
2062         case SAS_GPIO_REG_TX:
2063                 if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
2064                         dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
2065                                 reg_index, reg_index + reg_count - 1);
2066                         return -EINVAL;
2067                 }
2068
2069                 for (i = 0; i < reg_count; i++)
2070                         hisi_sas_write32(hisi_hba,
2071                                          SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
2072                                          data[i]);
2073                 break;
2074         default:
2075                 dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
2076                                 reg_type);
2077                 return -EINVAL;
2078         }
2079
2080         return 0;
2081 }
2082
2083 static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
2084                                              int delay_ms, int timeout_ms)
2085 {
2086         struct device *dev = hisi_hba->dev;
2087         int entries, entries_old = 0, time;
2088
2089         for (time = 0; time < timeout_ms; time += delay_ms) {
2090                 entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
2091                 if (entries == entries_old)
2092                         break;
2093
2094                 entries_old = entries;
2095                 msleep(delay_ms);
2096         }
2097
2098         dev_dbg(dev, "wait commands complete %dms\n", time);
2099 }
2100
2101 static struct scsi_host_template sht_v3_hw = {
2102         .name                   = DRV_NAME,
2103         .module                 = THIS_MODULE,
2104         .queuecommand           = sas_queuecommand,
2105         .target_alloc           = sas_target_alloc,
2106         .slave_configure        = hisi_sas_slave_configure,
2107         .scan_finished          = hisi_sas_scan_finished,
2108         .scan_start             = hisi_sas_scan_start,
2109         .change_queue_depth     = sas_change_queue_depth,
2110         .bios_param             = sas_bios_param,
2111         .can_queue              = 1,
2112         .this_id                = -1,
2113         .sg_tablesize           = SG_ALL,
2114         .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
2115         .use_clustering         = ENABLE_CLUSTERING,
2116         .eh_device_reset_handler = sas_eh_device_reset_handler,
2117         .eh_target_reset_handler = sas_eh_target_reset_handler,
2118         .slave_alloc            = sas_slave_alloc,
2119         .target_destroy         = sas_target_destroy,
2120         .ioctl                  = sas_ioctl,
2121         .shost_attrs            = host_attrs,
2122 };
2123
2124 static const struct hisi_sas_hw hisi_sas_v3_hw = {
2125         .hw_init = hisi_sas_v3_init,
2126         .setup_itct = setup_itct_v3_hw,
2127         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW,
2128         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
2129         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
2130         .clear_itct = clear_itct_v3_hw,
2131         .sl_notify_ssp = sl_notify_ssp_v3_hw,
2132         .prep_ssp = prep_ssp_v3_hw,
2133         .prep_smp = prep_smp_v3_hw,
2134         .prep_stp = prep_ata_v3_hw,
2135         .prep_abort = prep_abort_v3_hw,
2136         .get_free_slot = get_free_slot_v3_hw,
2137         .start_delivery = start_delivery_v3_hw,
2138         .slot_complete = slot_complete_v3_hw,
2139         .phys_init = phys_init_v3_hw,
2140         .phy_start = start_phy_v3_hw,
2141         .phy_disable = disable_phy_v3_hw,
2142         .phy_hard_reset = phy_hard_reset_v3_hw,
2143         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
2144         .phy_set_linkrate = phy_set_linkrate_v3_hw,
2145         .dereg_device = dereg_device_v3_hw,
2146         .soft_reset = soft_reset_v3_hw,
2147         .get_phys_state = get_phys_state_v3_hw,
2148         .get_events = phy_get_events_v3_hw,
2149         .write_gpio = write_gpio_v3_hw,
2150         .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
2151 };
2152
2153 static struct Scsi_Host *
2154 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
2155 {
2156         struct Scsi_Host *shost;
2157         struct hisi_hba *hisi_hba;
2158         struct device *dev = &pdev->dev;
2159
2160         shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
2161         if (!shost) {
2162                 dev_err(dev, "shost alloc failed\n");
2163                 return NULL;
2164         }
2165         hisi_hba = shost_priv(shost);
2166
2167         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2168         hisi_hba->hw = &hisi_sas_v3_hw;
2169         hisi_hba->pci_dev = pdev;
2170         hisi_hba->dev = dev;
2171         hisi_hba->shost = shost;
2172         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2173
2174         timer_setup(&hisi_hba->timer, NULL, 0);
2175
2176         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2177                 goto err_out;
2178
2179         if (hisi_sas_alloc(hisi_hba, shost)) {
2180                 hisi_sas_free(hisi_hba);
2181                 goto err_out;
2182         }
2183
2184         return shost;
2185 err_out:
2186         scsi_host_put(shost);
2187         dev_err(dev, "shost alloc failed\n");
2188         return NULL;
2189 }
2190
2191 static int
2192 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2193 {
2194         struct Scsi_Host *shost;
2195         struct hisi_hba *hisi_hba;
2196         struct device *dev = &pdev->dev;
2197         struct asd_sas_phy **arr_phy;
2198         struct asd_sas_port **arr_port;
2199         struct sas_ha_struct *sha;
2200         int rc, phy_nr, port_nr, i;
2201
2202         rc = pci_enable_device(pdev);
2203         if (rc)
2204                 goto err_out;
2205
2206         pci_set_master(pdev);
2207
2208         rc = pci_request_regions(pdev, DRV_NAME);
2209         if (rc)
2210                 goto err_out_disable_device;
2211
2212         if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) ||
2213             (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) {
2214                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) ||
2215                    (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) {
2216                         dev_err(dev, "No usable DMA addressing method\n");
2217                         rc = -EIO;
2218                         goto err_out_regions;
2219                 }
2220         }
2221
2222         shost = hisi_sas_shost_alloc_pci(pdev);
2223         if (!shost) {
2224                 rc = -ENOMEM;
2225                 goto err_out_regions;
2226         }
2227
2228         sha = SHOST_TO_SAS_HA(shost);
2229         hisi_hba = shost_priv(shost);
2230         dev_set_drvdata(dev, sha);
2231
2232         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
2233         if (!hisi_hba->regs) {
2234                 dev_err(dev, "cannot map register.\n");
2235                 rc = -ENOMEM;
2236                 goto err_out_ha;
2237         }
2238
2239         phy_nr = port_nr = hisi_hba->n_phy;
2240
2241         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2242         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2243         if (!arr_phy || !arr_port) {
2244                 rc = -ENOMEM;
2245                 goto err_out_ha;
2246         }
2247
2248         sha->sas_phy = arr_phy;
2249         sha->sas_port = arr_port;
2250         sha->core.shost = shost;
2251         sha->lldd_ha = hisi_hba;
2252
2253         shost->transportt = hisi_sas_stt;
2254         shost->max_id = HISI_SAS_MAX_DEVICES;
2255         shost->max_lun = ~0;
2256         shost->max_channel = 1;
2257         shost->max_cmd_len = 16;
2258         shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT);
2259         shost->can_queue = hisi_hba->hw->max_command_entries;
2260         shost->cmd_per_lun = hisi_hba->hw->max_command_entries;
2261
2262         sha->sas_ha_name = DRV_NAME;
2263         sha->dev = dev;
2264         sha->lldd_module = THIS_MODULE;
2265         sha->sas_addr = &hisi_hba->sas_addr[0];
2266         sha->num_phys = hisi_hba->n_phy;
2267         sha->core.shost = hisi_hba->shost;
2268
2269         for (i = 0; i < hisi_hba->n_phy; i++) {
2270                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2271                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2272         }
2273
2274         rc = scsi_add_host(shost, dev);
2275         if (rc)
2276                 goto err_out_ha;
2277
2278         rc = sas_register_ha(sha);
2279         if (rc)
2280                 goto err_out_register_ha;
2281
2282         rc = hisi_hba->hw->hw_init(hisi_hba);
2283         if (rc)
2284                 goto err_out_register_ha;
2285
2286         scsi_scan_host(shost);
2287
2288         return 0;
2289
2290 err_out_register_ha:
2291         scsi_remove_host(shost);
2292 err_out_ha:
2293         scsi_host_put(shost);
2294 err_out_regions:
2295         pci_release_regions(pdev);
2296 err_out_disable_device:
2297         pci_disable_device(pdev);
2298 err_out:
2299         return rc;
2300 }
2301
2302 static void
2303 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
2304 {
2305         int i;
2306
2307         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
2308         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
2309         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
2310         for (i = 0; i < hisi_hba->queue_count; i++) {
2311                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2312
2313                 free_irq(pci_irq_vector(pdev, i+16), cq);
2314         }
2315         pci_free_irq_vectors(pdev);
2316 }
2317
2318 static void hisi_sas_v3_remove(struct pci_dev *pdev)
2319 {
2320         struct device *dev = &pdev->dev;
2321         struct sas_ha_struct *sha = dev_get_drvdata(dev);
2322         struct hisi_hba *hisi_hba = sha->lldd_ha;
2323         struct Scsi_Host *shost = sha->core.shost;
2324
2325         if (timer_pending(&hisi_hba->timer))
2326                 del_timer(&hisi_hba->timer);
2327
2328         sas_unregister_ha(sha);
2329         sas_remove_host(sha->core.shost);
2330
2331         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
2332         hisi_sas_kill_tasklets(hisi_hba);
2333         pci_release_regions(pdev);
2334         pci_disable_device(pdev);
2335         hisi_sas_free(hisi_hba);
2336         scsi_host_put(shost);
2337 }
2338
2339 static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
2340         { .irq_msk = BIT(19), .msg = "HILINK_INT" },
2341         { .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
2342         { .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
2343         { .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
2344         { .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
2345         { .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
2346         { .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
2347         { .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
2348         { .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
2349         { .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
2350         { .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
2351         { .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
2352         { .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
2353 };
2354
2355 static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
2356         { .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
2357         { .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
2358         { .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
2359         { .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
2360         { .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
2361         { .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
2362         { .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
2363         { .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
2364         { .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
2365         { .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
2366         { .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
2367         { .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
2368         { .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
2369         { .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
2370         { .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
2371         { .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
2372         { .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
2373         { .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
2374         { .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
2375         { .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
2376         { .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
2377         { .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
2378         { .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
2379         { .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
2380         { .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
2381         { .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
2382         { .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
2383         { .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
2384         { .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
2385         { .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
2386         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
2387 };
2388
2389 static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
2390         { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
2391         { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
2392         { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
2393         { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
2394         { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
2395         { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
2396         { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
2397         { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
2398         { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
2399         { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
2400         { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
2401         { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
2402         { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
2403         { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
2404         { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
2405         { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
2406         { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
2407         { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
2408         { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
2409         { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
2410 };
2411
2412 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
2413 {
2414         struct device *dev = hisi_hba->dev;
2415         const struct hisi_sas_hw_error *ras_error;
2416         bool need_reset = false;
2417         u32 irq_value;
2418         int i;
2419
2420         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
2421         for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
2422                 ras_error = &sas_ras_intr0_nfe[i];
2423                 if (ras_error->irq_msk & irq_value) {
2424                         dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
2425                                         ras_error->msg, irq_value);
2426                         need_reset = true;
2427                 }
2428         }
2429         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
2430
2431         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
2432         for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
2433                 ras_error = &sas_ras_intr1_nfe[i];
2434                 if (ras_error->irq_msk & irq_value) {
2435                         dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
2436                                         ras_error->msg, irq_value);
2437                         need_reset = true;
2438                 }
2439         }
2440         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
2441
2442         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
2443         for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
2444                 ras_error = &sas_ras_intr2_nfe[i];
2445                 if (ras_error->irq_msk & irq_value) {
2446                         dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
2447                                         ras_error->msg, irq_value);
2448                         need_reset = true;
2449                 }
2450         }
2451         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
2452
2453         return need_reset;
2454 }
2455
2456 static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
2457                 pci_channel_state_t state)
2458 {
2459         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2460         struct hisi_hba *hisi_hba = sha->lldd_ha;
2461         struct device *dev = hisi_hba->dev;
2462
2463         dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
2464         if (state == pci_channel_io_perm_failure)
2465                 return PCI_ERS_RESULT_DISCONNECT;
2466
2467         if (process_non_fatal_error_v3_hw(hisi_hba))
2468                 return PCI_ERS_RESULT_NEED_RESET;
2469
2470         return PCI_ERS_RESULT_CAN_RECOVER;
2471 }
2472
2473 static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
2474 {
2475         return PCI_ERS_RESULT_RECOVERED;
2476 }
2477
2478 static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
2479 {
2480         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2481         struct hisi_hba *hisi_hba = sha->lldd_ha;
2482         struct device *dev = hisi_hba->dev;
2483         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
2484
2485         dev_info(dev, "PCI error: slot reset callback!!\n");
2486         queue_work(hisi_hba->wq, &r.work);
2487         wait_for_completion(r.completion);
2488         if (r.done)
2489                 return PCI_ERS_RESULT_RECOVERED;
2490
2491         return PCI_ERS_RESULT_DISCONNECT;
2492 }
2493
2494 static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
2495 {
2496         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2497         struct hisi_hba *hisi_hba = sha->lldd_ha;
2498         struct device *dev = hisi_hba->dev;
2499         int rc;
2500
2501         dev_info(dev, "FLR prepare\n");
2502         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2503         hisi_sas_controller_reset_prepare(hisi_hba);
2504
2505         rc = disable_host_v3_hw(hisi_hba);
2506         if (rc)
2507                 dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
2508 }
2509
2510 static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev)
2511 {
2512         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2513         struct hisi_hba *hisi_hba = sha->lldd_ha;
2514         struct device *dev = hisi_hba->dev;
2515         int rc;
2516
2517         hisi_sas_init_mem(hisi_hba);
2518
2519         rc = hw_init_v3_hw(hisi_hba);
2520         if (rc) {
2521                 dev_err(dev, "FLR: hw init failed rc=%d\n", rc);
2522                 return;
2523         }
2524
2525         hisi_sas_controller_reset_done(hisi_hba);
2526         dev_info(dev, "FLR done\n");
2527 }
2528
2529 enum {
2530         /* instances of the controller */
2531         hip08,
2532 };
2533
2534 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
2535 {
2536         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2537         struct hisi_hba *hisi_hba = sha->lldd_ha;
2538         struct device *dev = hisi_hba->dev;
2539         struct Scsi_Host *shost = hisi_hba->shost;
2540         u32 device_state;
2541         int rc;
2542
2543         if (!pdev->pm_cap) {
2544                 dev_err(dev, "PCI PM not supported\n");
2545                 return -ENODEV;
2546         }
2547
2548         if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
2549                 return -1;
2550
2551         scsi_block_requests(shost);
2552         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2553         flush_workqueue(hisi_hba->wq);
2554
2555         rc = disable_host_v3_hw(hisi_hba);
2556         if (rc) {
2557                 dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
2558                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2559                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2560                 scsi_unblock_requests(shost);
2561                 return rc;
2562         }
2563
2564         hisi_sas_init_mem(hisi_hba);
2565
2566         device_state = pci_choose_state(pdev, state);
2567         dev_warn(dev, "entering operating state [D%d]\n",
2568                         device_state);
2569         pci_save_state(pdev);
2570         pci_disable_device(pdev);
2571         pci_set_power_state(pdev, device_state);
2572
2573         hisi_sas_release_tasks(hisi_hba);
2574
2575         sas_suspend_ha(sha);
2576         return 0;
2577 }
2578
2579 static int hisi_sas_v3_resume(struct pci_dev *pdev)
2580 {
2581         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2582         struct hisi_hba *hisi_hba = sha->lldd_ha;
2583         struct Scsi_Host *shost = hisi_hba->shost;
2584         struct device *dev = hisi_hba->dev;
2585         unsigned int rc;
2586         u32 device_state = pdev->current_state;
2587
2588         dev_warn(dev, "resuming from operating state [D%d]\n",
2589                         device_state);
2590         pci_set_power_state(pdev, PCI_D0);
2591         pci_enable_wake(pdev, PCI_D0, 0);
2592         pci_restore_state(pdev);
2593         rc = pci_enable_device(pdev);
2594         if (rc)
2595                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
2596
2597         pci_set_master(pdev);
2598         scsi_unblock_requests(shost);
2599         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2600
2601         sas_prep_resume_ha(sha);
2602         init_reg_v3_hw(hisi_hba);
2603         hisi_hba->hw->phys_init(hisi_hba);
2604         sas_resume_ha(sha);
2605         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2606
2607         return 0;
2608 }
2609
2610 static const struct pci_device_id sas_v3_pci_table[] = {
2611         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
2612         {}
2613 };
2614 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
2615
2616 static const struct pci_error_handlers hisi_sas_err_handler = {
2617         .error_detected = hisi_sas_error_detected_v3_hw,
2618         .mmio_enabled   = hisi_sas_mmio_enabled_v3_hw,
2619         .slot_reset     = hisi_sas_slot_reset_v3_hw,
2620         .reset_prepare  = hisi_sas_reset_prepare_v3_hw,
2621         .reset_done     = hisi_sas_reset_done_v3_hw,
2622 };
2623
2624 static struct pci_driver sas_v3_pci_driver = {
2625         .name           = DRV_NAME,
2626         .id_table       = sas_v3_pci_table,
2627         .probe          = hisi_sas_v3_probe,
2628         .remove         = hisi_sas_v3_remove,
2629         .suspend        = hisi_sas_v3_suspend,
2630         .resume         = hisi_sas_v3_resume,
2631         .err_handler    = &hisi_sas_err_handler,
2632 };
2633
2634 module_pci_driver(sas_v3_pci_driver);
2635
2636 MODULE_LICENSE("GPL");
2637 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
2638 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
2639 MODULE_ALIAS("pci:" DRV_NAME);