drivers/scsi/aic7xxx/aic79xx_core.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2003 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
  41  */
  42
  43 #include "aic79xx_osm.h"
  44 #include "aic79xx_inline.h"
  45 #include "aicasm/aicasm_insformat.h"
  46
  47 /***************************** Lookup Tables **********************************/
  48 static const char *const ahd_chip_names[] =
  49 {
  50         "NONE",
  51         "aic7901",
  52         "aic7902",
  53         "aic7901A"
  54 };
  55
  56 /*
  57  * Hardware error codes.
  58  */
  59 struct ahd_hard_error_entry {
  60         uint8_t errno;
  61         const char *errmesg;
  62 };
  63
  64 static const struct ahd_hard_error_entry ahd_hard_errors[] = {
  65         { DSCTMOUT,     "Discard Timer has timed out" },
  66         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  67         { SQPARERR,     "Sequencer Parity Error" },
  68         { DPARERR,      "Data-path Parity Error" },
  69         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  70         { CIOPARERR,    "CIOBUS Parity Error" },
  71 };
  72 static const u_int num_errors = ARRAY_SIZE(ahd_hard_errors);
  73
  74 static const struct ahd_phase_table_entry ahd_phase_table[] =
  75 {
  76         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  77         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  78         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  79         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  80         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  81         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  82         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  83         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  84         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  85         { 0,            MSG_NOOP,               "in unknown phase"      }
  86 };
  87
  88 /*
  89  * In most cases we only wish to itterate over real phases, so
  90  * exclude the last element from the count.
  91  */
  92 static const u_int num_phases = ARRAY_SIZE(ahd_phase_table) - 1;
  93
  94 /* Our Sequencer Program */
  95 #include "aic79xx_seq.h"
  96
  97 /**************************** Function Declarations ***************************/
  98 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
  99 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 100                                                   u_int lqistat1);
 101 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 102                                                u_int busfreetime);
 103 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 104 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 105 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 106                                                 struct ahd_devinfo *devinfo);
 107
 108 static struct ahd_tmode_tstate*
 109                         ahd_alloc_tstate(struct ahd_softc *ahd,
 110                                          u_int scsi_id, char channel);
 111 #ifdef AHD_TARGET_MODE
 112 static void             ahd_free_tstate(struct ahd_softc *ahd,
 113                                         u_int scsi_id, char channel, int force);
 114 #endif
 115 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 116                                                 struct ahd_initiator_tinfo *,
 117                                                 u_int *period,
 118                                                 u_int *ppr_options,
 119                                                 role_t role);
 120 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 121                                              struct ahd_devinfo *devinfo,
 122                                              struct ahd_transinfo *tinfo);
 123 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 124 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 125                                           struct ahd_devinfo *devinfo);
 126 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 127                                         struct ahd_devinfo *devinfo,
 128                                         struct scb *scb);
 129 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 130                                                    struct ahd_devinfo *devinfo,
 131                                                    struct scb *scb);
 132 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 133                                                struct ahd_devinfo *devinfo);
 134 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 135                                            struct ahd_devinfo *devinfo,
 136                                            u_int period, u_int offset);
 137 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 138                                            struct ahd_devinfo *devinfo,
 139                                            u_int bus_width);
 140 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 141                                           struct ahd_devinfo *devinfo,
 142                                           u_int period, u_int offset,
 143                                           u_int bus_width, u_int ppr_options);
 144 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 145 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 146 typedef enum {
 147         AHDMSG_1B,
 148         AHDMSG_2B,
 149         AHDMSG_EXT
 150 } ahd_msgtype;
 151 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 152                                      u_int msgval, int full);
 153 static int              ahd_parse_msg(struct ahd_softc *ahd,
 154                                       struct ahd_devinfo *devinfo);
 155 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 156                                               struct ahd_devinfo *devinfo);
 157 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 158                                                 struct ahd_devinfo *devinfo);
 159 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 160 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 161                                             struct ahd_devinfo *devinfo,
 162                                             u_int lun, cam_status status,
 163                                             char *message, int verbose_level);
 164 #ifdef AHD_TARGET_MODE
 165 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 166                                                struct ahd_devinfo *devinfo,
 167                                                struct scb *scb);
 168 #endif
 169
 170 static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 171 static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 172 static bus_dmamap_callback_t
 173                         ahd_dmamap_cb;
 174 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 175 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 176 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 177 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 178 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 179 static void             ahd_add_col_list(struct ahd_softc *ahd,
 180                                          struct scb *scb, u_int col_idx);
 181 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 182                                          struct scb *scb);
 183 static void             ahd_chip_init(struct ahd_softc *ahd);
 184 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 185                                             struct scb *prev_scb,
 186                                             struct scb *scb);
 187 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 188 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 189                                             char channel, int lun, u_int tag,
 190                                             role_t role, uint32_t status,
 191                                             ahd_search_action action,
 192                                             u_int *list_head, u_int *list_tail,
 193                                             u_int tid);
 194 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 195                                             u_int tid_prev, u_int tid_cur,
 196                                             u_int tid_next);
 197 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 198                                                  u_int scbid);
 199 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 200                                      u_int prev, u_int next, u_int tid);
 201 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 202 static void             ahd_stat_timer(struct timer_list *t);
 203 #ifdef AHD_DUMP_SEQ
 204 static void             ahd_dumpseq(struct ahd_softc *ahd);
 205 #endif
 206 static void             ahd_loadseq(struct ahd_softc *ahd);
 207 static int              ahd_check_patch(struct ahd_softc *ahd,
 208                                         const struct patch **start_patch,
 209                                         u_int start_instr, u_int *skip_addr);
 210 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 211                                             u_int address);
 212 static void             ahd_download_instr(struct ahd_softc *ahd,
 213                                            u_int instrptr, uint8_t *dconsts);
 214 static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 215 static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 216                                                struct scb *scb);
 217 static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 218                                           struct scb *scb);
 219
 220 #ifdef AHD_TARGET_MODE
 221 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 222                                                struct ahd_tmode_lstate *lstate,
 223                                                u_int initiator_id,
 224                                                u_int event_type,
 225                                                u_int event_arg);
 226 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 227                                           u_int targid_mask);
 228 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 229                                               struct target_cmd *cmd);
 230 #endif
 231
 232 static int              ahd_abort_scbs(struct ahd_softc *ahd, int target,
 233                                        char channel, int lun, u_int tag,
 234                                        role_t role, uint32_t status);
 235 static void             ahd_alloc_scbs(struct ahd_softc *ahd);
 236 static void             ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl,
 237                                      u_int scbid);
 238 static void             ahd_calc_residual(struct ahd_softc *ahd,
 239                                           struct scb *scb);
 240 static void             ahd_clear_critical_section(struct ahd_softc *ahd);
 241 static void             ahd_clear_intstat(struct ahd_softc *ahd);
 242 static void             ahd_enable_coalescing(struct ahd_softc *ahd,
 243                                               int enable);
 244 static u_int            ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl);
 245 static void             ahd_freeze_devq(struct ahd_softc *ahd,
 246                                         struct scb *scb);
 247 static void             ahd_handle_scb_status(struct ahd_softc *ahd,
 248                                               struct scb *scb);
 249 static const struct ahd_phase_table_entry* ahd_lookup_phase_entry(int phase);
 250 static void             ahd_shutdown(void *arg);
 251 static void             ahd_update_coalescing_values(struct ahd_softc *ahd,
 252                                                      u_int timer,
 253                                                      u_int maxcmds,
 254                                                      u_int mincmds);
 255 static int              ahd_verify_vpd_cksum(struct vpd_config *vpd);
 256 static int              ahd_wait_seeprom(struct ahd_softc *ahd);
 257 static int              ahd_match_scb(struct ahd_softc *ahd, struct scb *scb,
 258                                       int target, char channel, int lun,
 259                                       u_int tag, role_t role);
 260
 261 static void             ahd_reset_cmds_pending(struct ahd_softc *ahd);
 262
 263 /*************************** Interrupt Services *******************************/
 264 static void             ahd_run_qoutfifo(struct ahd_softc *ahd);
 265 #ifdef AHD_TARGET_MODE
 266 static void             ahd_run_tqinfifo(struct ahd_softc *ahd, int paused);
 267 #endif
 268 static void             ahd_handle_hwerrint(struct ahd_softc *ahd);
 269 static void             ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat);
 270 static void             ahd_handle_scsiint(struct ahd_softc *ahd,
 271                                            u_int intstat);
 272
 273 /************************ Sequencer Execution Control *************************/
 274 void
 275 ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst)
 276 {
 277         if (ahd->src_mode == src && ahd->dst_mode == dst)
 278                 return;
 279 #ifdef AHD_DEBUG
 280         if (ahd->src_mode == AHD_MODE_UNKNOWN
 281          || ahd->dst_mode == AHD_MODE_UNKNOWN)
 282                 panic("Setting mode prior to saving it.\n");
 283         if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 284                 printk("%s: Setting mode 0x%x\n", ahd_name(ahd),
 285                        ahd_build_mode_state(ahd, src, dst));
 286 #endif
 287         ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst));
 288         ahd->src_mode = src;
 289         ahd->dst_mode = dst;
 290 }
 291
 292 static void
 293 ahd_update_modes(struct ahd_softc *ahd)
 294 {
 295         ahd_mode_state mode_ptr;
 296         ahd_mode src;
 297         ahd_mode dst;
 298
 299         mode_ptr = ahd_inb(ahd, MODE_PTR);
 300 #ifdef AHD_DEBUG
 301         if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 302                 printk("Reading mode 0x%x\n", mode_ptr);
 303 #endif
 304         ahd_extract_mode_state(ahd, mode_ptr, &src, &dst);
 305         ahd_known_modes(ahd, src, dst);
 306 }
 307
 308 static void
 309 ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode,
 310                  ahd_mode dstmode, const char *file, int line)
 311 {
 312 #ifdef AHD_DEBUG
 313         if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0
 314          || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) {
 315                 panic("%s:%s:%d: Mode assertion failed.\n",
 316                        ahd_name(ahd), file, line);
 317         }
 318 #endif
 319 }
 320
 321 #define AHD_ASSERT_MODES(ahd, source, dest) \
 322         ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__);
 323
 324 ahd_mode_state
 325 ahd_save_modes(struct ahd_softc *ahd)
 326 {
 327         if (ahd->src_mode == AHD_MODE_UNKNOWN
 328          || ahd->dst_mode == AHD_MODE_UNKNOWN)
 329                 ahd_update_modes(ahd);
 330
 331         return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode));
 332 }
 333
 334 void
 335 ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state)
 336 {
 337         ahd_mode src;
 338         ahd_mode dst;
 339
 340         ahd_extract_mode_state(ahd, state, &src, &dst);
 341         ahd_set_modes(ahd, src, dst);
 342 }
 343
 344 /*
 345  * Determine whether the sequencer has halted code execution.
 346  * Returns non-zero status if the sequencer is stopped.
 347  */
 348 int
 349 ahd_is_paused(struct ahd_softc *ahd)
 350 {
 351         return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0);
 352 }
 353
 354 /*
 355  * Request that the sequencer stop and wait, indefinitely, for it
 356  * to stop.  The sequencer will only acknowledge that it is paused
 357  * once it has reached an instruction boundary and PAUSEDIS is
 358  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 359  * for critical sections.
 360  */
 361 void
 362 ahd_pause(struct ahd_softc *ahd)
 363 {
 364         ahd_outb(ahd, HCNTRL, ahd->pause);
 365
 366         /*
 367          * Since the sequencer can disable pausing in a critical section, we
 368          * must loop until it actually stops.
 369          */
 370         while (ahd_is_paused(ahd) == 0)
 371                 ;
 372 }
 373
 374 /*
 375  * Allow the sequencer to continue program execution.
 376  * We check here to ensure that no additional interrupt
 377  * sources that would cause the sequencer to halt have been
 378  * asserted.  If, for example, a SCSI bus reset is detected
 379  * while we are fielding a different, pausing, interrupt type,
 380  * we don't want to release the sequencer before going back
 381  * into our interrupt handler and dealing with this new
 382  * condition.
 383  */
 384 void
 385 ahd_unpause(struct ahd_softc *ahd)
 386 {
 387         /*
 388          * Automatically restore our modes to those saved
 389          * prior to the first change of the mode.
 390          */
 391         if (ahd->saved_src_mode != AHD_MODE_UNKNOWN
 392          && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) {
 393                 if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0)
 394                         ahd_reset_cmds_pending(ahd);
 395                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
 396         }
 397
 398         if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0)
 399                 ahd_outb(ahd, HCNTRL, ahd->unpause);
 400
 401         ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN);
 402 }
 403
 404 /*********************** Scatter Gather List Handling *************************/
 405 void *
 406 ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb,
 407              void *sgptr, dma_addr_t addr, bus_size_t len, int last)
 408 {
 409         scb->sg_count++;
 410         if (sizeof(dma_addr_t) > 4
 411          && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 412                 struct ahd_dma64_seg *sg;
 413
 414                 sg = (struct ahd_dma64_seg *)sgptr;
 415                 sg->addr = ahd_htole64(addr);
 416                 sg->len = ahd_htole32(len | (last ? AHD_DMA_LAST_SEG : 0));
 417                 return (sg + 1);
 418         } else {
 419                 struct ahd_dma_seg *sg;
 420
 421                 sg = (struct ahd_dma_seg *)sgptr;
 422                 sg->addr = ahd_htole32(addr & 0xFFFFFFFF);
 423                 sg->len = ahd_htole32(len | ((addr >> 8) & 0x7F000000)
 424                                     | (last ? AHD_DMA_LAST_SEG : 0));
 425                 return (sg + 1);
 426         }
 427 }
 428
 429 static void
 430 ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb)
 431 {
 432         /* XXX Handle target mode SCBs. */
 433         scb->crc_retry_count = 0;
 434         if ((scb->flags & SCB_PACKETIZED) != 0) {
 435                 /* XXX what about ACA??  It is type 4, but TAG_TYPE == 0x3. */
 436                 scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE;
 437         } else {
 438                 if (ahd_get_transfer_length(scb) & 0x01)
 439                         scb->hscb->task_attribute = SCB_XFERLEN_ODD;
 440                 else
 441                         scb->hscb->task_attribute = 0;
 442         }
 443
 444         if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR
 445          || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0)
 446                 scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr =
 447                     ahd_htole32(scb->sense_busaddr);
 448 }
 449
 450 static void
 451 ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb)
 452 {
 453         /*
 454          * Copy the first SG into the "current" data ponter area.
 455          */
 456         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 457                 struct ahd_dma64_seg *sg;
 458
 459                 sg = (struct ahd_dma64_seg *)scb->sg_list;
 460                 scb->hscb->dataptr = sg->addr;
 461                 scb->hscb->datacnt = sg->len;
 462         } else {
 463                 struct ahd_dma_seg *sg;
 464                 uint32_t *dataptr_words;
 465
 466                 sg = (struct ahd_dma_seg *)scb->sg_list;
 467                 dataptr_words = (uint32_t*)&scb->hscb->dataptr;
 468                 dataptr_words[0] = sg->addr;
 469                 dataptr_words[1] = 0;
 470                 if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) {
 471                         uint64_t high_addr;
 472
 473                         high_addr = ahd_le32toh(sg->len) & 0x7F000000;
 474                         scb->hscb->dataptr |= ahd_htole64(high_addr << 8);
 475                 }
 476                 scb->hscb->datacnt = sg->len;
 477         }
 478         /*
 479          * Note where to find the SG entries in bus space.
 480          * We also set the full residual flag which the
 481          * sequencer will clear as soon as a data transfer
 482          * occurs.
 483          */
 484         scb->hscb->sgptr = ahd_htole32(scb->sg_list_busaddr|SG_FULL_RESID);
 485 }
 486
 487 static void
 488 ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb)
 489 {
 490         scb->hscb->sgptr = ahd_htole32(SG_LIST_NULL);
 491         scb->hscb->dataptr = 0;
 492         scb->hscb->datacnt = 0;
 493 }
 494
 495 /************************** Memory mapping routines ***************************/
 496 static void *
 497 ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr)
 498 {
 499         dma_addr_t sg_offset;
 500
 501         /* sg_list_phys points to entry 1, not 0 */
 502         sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd));
 503         return ((uint8_t *)scb->sg_list + sg_offset);
 504 }
 505
 506 static uint32_t
 507 ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg)
 508 {
 509         dma_addr_t sg_offset;
 510
 511         /* sg_list_phys points to entry 1, not 0 */
 512         sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list)
 513                   - ahd_sg_size(ahd);
 514
 515         return (scb->sg_list_busaddr + sg_offset);
 516 }
 517
 518 static void
 519 ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op)
 520 {
 521         ahd_dmamap_sync(ahd, ahd->scb_data.hscb_dmat,
 522                         scb->hscb_map->dmamap,
 523                         /*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr,
 524                         /*len*/sizeof(*scb->hscb), op);
 525 }
 526
 527 void
 528 ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op)
 529 {
 530         if (scb->sg_count == 0)
 531                 return;
 532
 533         ahd_dmamap_sync(ahd, ahd->scb_data.sg_dmat,
 534                         scb->sg_map->dmamap,
 535                         /*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd),
 536                         /*len*/ahd_sg_size(ahd) * scb->sg_count, op);
 537 }
 538
 539 static void
 540 ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op)
 541 {
 542         ahd_dmamap_sync(ahd, ahd->scb_data.sense_dmat,
 543                         scb->sense_map->dmamap,
 544                         /*offset*/scb->sense_busaddr,
 545                         /*len*/AHD_SENSE_BUFSIZE, op);
 546 }
 547
 548 #ifdef AHD_TARGET_MODE
 549 static uint32_t
 550 ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index)
 551 {
 552         return (((uint8_t *)&ahd->targetcmds[index])
 553                - (uint8_t *)ahd->qoutfifo);
 554 }
 555 #endif
 556
 557 /*********************** Miscellaneous Support Functions ***********************/
 558 /*
 559  * Return pointers to the transfer negotiation information
 560  * for the specified our_id/remote_id pair.
 561  */
 562 struct ahd_initiator_tinfo *
 563 ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id,
 564                     u_int remote_id, struct ahd_tmode_tstate **tstate)
 565 {
 566         /*
 567          * Transfer data structures are stored from the perspective
 568          * of the target role.  Since the parameters for a connection
 569          * in the initiator role to a given target are the same as
 570          * when the roles are reversed, we pretend we are the target.
 571          */
 572         if (channel == 'B')
 573                 our_id += 8;
 574         *tstate = ahd->enabled_targets[our_id];
 575         return (&(*tstate)->transinfo[remote_id]);
 576 }
 577
 578 uint16_t
 579 ahd_inw(struct ahd_softc *ahd, u_int port)
 580 {
 581         /*
 582          * Read high byte first as some registers increment
 583          * or have other side effects when the low byte is
 584          * read.
 585          */
 586         uint16_t r = ahd_inb(ahd, port+1) << 8;
 587         return r | ahd_inb(ahd, port);
 588 }
 589
 590 void
 591 ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
 592 {
 593         /*
 594          * Write low byte first to accommodate registers
 595          * such as PRGMCNT where the order maters.
 596          */
 597         ahd_outb(ahd, port, value & 0xFF);
 598         ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 599 }
 600
 601 uint32_t
 602 ahd_inl(struct ahd_softc *ahd, u_int port)
 603 {
 604         return ((ahd_inb(ahd, port))
 605               | (ahd_inb(ahd, port+1) << 8)
 606               | (ahd_inb(ahd, port+2) << 16)
 607               | (ahd_inb(ahd, port+3) << 24));
 608 }
 609
 610 void
 611 ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
 612 {
 613         ahd_outb(ahd, port, (value) & 0xFF);
 614         ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
 615         ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
 616         ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
 617 }
 618
 619 uint64_t
 620 ahd_inq(struct ahd_softc *ahd, u_int port)
 621 {
 622         return ((ahd_inb(ahd, port))
 623               | (ahd_inb(ahd, port+1) << 8)
 624               | (ahd_inb(ahd, port+2) << 16)
 625               | (ahd_inb(ahd, port+3) << 24)
 626               | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
 627               | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
 628               | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
 629               | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
 630 }
 631
 632 void
 633 ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
 634 {
 635         ahd_outb(ahd, port, value & 0xFF);
 636         ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 637         ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
 638         ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
 639         ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
 640         ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
 641         ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
 642         ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
 643 }
 644
 645 u_int
 646 ahd_get_scbptr(struct ahd_softc *ahd)
 647 {
 648         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 649                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 650         return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
 651 }
 652
 653 void
 654 ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
 655 {
 656         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 657                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 658         ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
 659         ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
 660 }
 661
 662 #if 0 /* unused */
 663 static u_int
 664 ahd_get_hnscb_qoff(struct ahd_softc *ahd)
 665 {
 666         return (ahd_inw_atomic(ahd, HNSCB_QOFF));
 667 }
 668 #endif
 669
 670 static void
 671 ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
 672 {
 673         ahd_outw_atomic(ahd, HNSCB_QOFF, value);
 674 }
 675
 676 #if 0 /* unused */
 677 static u_int
 678 ahd_get_hescb_qoff(struct ahd_softc *ahd)
 679 {
 680         return (ahd_inb(ahd, HESCB_QOFF));
 681 }
 682 #endif
 683
 684 static void
 685 ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
 686 {
 687         ahd_outb(ahd, HESCB_QOFF, value);
 688 }
 689
 690 static u_int
 691 ahd_get_snscb_qoff(struct ahd_softc *ahd)
 692 {
 693         u_int oldvalue;
 694
 695         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 696         oldvalue = ahd_inw(ahd, SNSCB_QOFF);
 697         ahd_outw(ahd, SNSCB_QOFF, oldvalue);
 698         return (oldvalue);
 699 }
 700
 701 static void
 702 ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
 703 {
 704         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 705         ahd_outw(ahd, SNSCB_QOFF, value);
 706 }
 707
 708 #if 0 /* unused */
 709 static u_int
 710 ahd_get_sescb_qoff(struct ahd_softc *ahd)
 711 {
 712         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 713         return (ahd_inb(ahd, SESCB_QOFF));
 714 }
 715 #endif
 716
 717 static void
 718 ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
 719 {
 720         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 721         ahd_outb(ahd, SESCB_QOFF, value);
 722 }
 723
 724 #if 0 /* unused */
 725 static u_int
 726 ahd_get_sdscb_qoff(struct ahd_softc *ahd)
 727 {
 728         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 729         return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
 730 }
 731 #endif
 732
 733 static void
 734 ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
 735 {
 736         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 737         ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
 738         ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
 739 }
 740
 741 u_int
 742 ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
 743 {
 744         u_int value;
 745
 746         /*
 747          * Workaround PCI-X Rev A. hardware bug.
 748          * After a host read of SCB memory, the chip
 749          * may become confused into thinking prefetch
 750          * was required.  This starts the discard timer
 751          * running and can cause an unexpected discard
 752          * timer interrupt.  The work around is to read
 753          * a normal register prior to the exhaustion of
 754          * the discard timer.  The mode pointer register
 755          * has no side effects and so serves well for
 756          * this purpose.
 757          *
 758          * Razor #528
 759          */
 760         value = ahd_inb(ahd, offset);
 761         if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
 762                 ahd_inb(ahd, MODE_PTR);
 763         return (value);
 764 }
 765
 766 u_int
 767 ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
 768 {
 769         return (ahd_inb_scbram(ahd, offset)
 770               | (ahd_inb_scbram(ahd, offset+1) << 8));
 771 }
 772
 773 static uint32_t
 774 ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
 775 {
 776         return (ahd_inw_scbram(ahd, offset)
 777               | (ahd_inw_scbram(ahd, offset+2) << 16));
 778 }
 779
 780 static uint64_t
 781 ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
 782 {
 783         return (ahd_inl_scbram(ahd, offset)
 784               | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
 785 }
 786
 787 struct scb *
 788 ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
 789 {
 790         struct scb* scb;
 791
 792         if (tag >= AHD_SCB_MAX)
 793                 return (NULL);
 794         scb = ahd->scb_data.scbindex[tag];
 795         if (scb != NULL)
 796                 ahd_sync_scb(ahd, scb,
 797                              BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 798         return (scb);
 799 }
 800
 801 static void
 802 ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
 803 {
 804         struct   hardware_scb *q_hscb;
 805         struct   map_node *q_hscb_map;
 806         uint32_t saved_hscb_busaddr;
 807
 808         /*
 809          * Our queuing method is a bit tricky.  The card
 810          * knows in advance which HSCB (by address) to download,
 811          * and we can't disappoint it.  To achieve this, the next
 812          * HSCB to download is saved off in ahd->next_queued_hscb.
 813          * When we are called to queue "an arbitrary scb",
 814          * we copy the contents of the incoming HSCB to the one
 815          * the sequencer knows about, swap HSCB pointers and
 816          * finally assign the SCB to the tag indexed location
 817          * in the scb_array.  This makes sure that we can still
 818          * locate the correct SCB by SCB_TAG.
 819          */
 820         q_hscb = ahd->next_queued_hscb;
 821         q_hscb_map = ahd->next_queued_hscb_map;
 822         saved_hscb_busaddr = q_hscb->hscb_busaddr;
 823         memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 824         q_hscb->hscb_busaddr = saved_hscb_busaddr;
 825         q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
 826
 827         /* Now swap HSCB pointers. */
 828         ahd->next_queued_hscb = scb->hscb;
 829         ahd->next_queued_hscb_map = scb->hscb_map;
 830         scb->hscb = q_hscb;
 831         scb->hscb_map = q_hscb_map;
 832
 833         /* Now define the mapping from tag to SCB in the scbindex */
 834         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
 835 }
 836
 837 /*
 838  * Tell the sequencer about a new transaction to execute.
 839  */
 840 void
 841 ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
 842 {
 843         ahd_swap_with_next_hscb(ahd, scb);
 844
 845         if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
 846                 panic("Attempt to queue invalid SCB tag %x\n",
 847                       SCB_GET_TAG(scb));
 848
 849         /*
 850          * Keep a history of SCBs we've downloaded in the qinfifo.
 851          */
 852         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
 853         ahd->qinfifonext++;
 854
 855         if (scb->sg_count != 0)
 856                 ahd_setup_data_scb(ahd, scb);
 857         else
 858                 ahd_setup_noxfer_scb(ahd, scb);
 859         ahd_setup_scb_common(ahd, scb);
 860
 861         /*
 862          * Make sure our data is consistent from the
 863          * perspective of the adapter.
 864          */
 865         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 866
 867 #ifdef AHD_DEBUG
 868         if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
 869                 uint64_t host_dataptr;
 870
 871                 host_dataptr = ahd_le64toh(scb->hscb->dataptr);
 872                 printk("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
 873                        ahd_name(ahd),
 874                        SCB_GET_TAG(scb), scb->hscb->scsiid,
 875                        ahd_le32toh(scb->hscb->hscb_busaddr),
 876                        (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
 877                        (u_int)(host_dataptr & 0xFFFFFFFF),
 878                        ahd_le32toh(scb->hscb->datacnt));
 879         }
 880 #endif
 881         /* Tell the adapter about the newly queued SCB */
 882         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
 883 }
 884
 885 /************************** Interrupt Processing ******************************/
 886 static void
 887 ahd_sync_qoutfifo(struct ahd_softc *ahd, int op)
 888 {
 889         ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 890                         /*offset*/0,
 891                         /*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op);
 892 }
 893
 894 static void
 895 ahd_sync_tqinfifo(struct ahd_softc *ahd, int op)
 896 {
 897 #ifdef AHD_TARGET_MODE
 898         if ((ahd->flags & AHD_TARGETROLE) != 0) {
 899                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 900                                 ahd->shared_data_map.dmamap,
 901                                 ahd_targetcmd_offset(ahd, 0),
 902                                 sizeof(struct target_cmd) * AHD_TMODE_CMDS,
 903                                 op);
 904         }
 905 #endif
 906 }
 907
 908 /*
 909  * See if the firmware has posted any completed commands
 910  * into our in-core command complete fifos.
 911  */
 912 #define AHD_RUN_QOUTFIFO 0x1
 913 #define AHD_RUN_TQINFIFO 0x2
 914 static u_int
 915 ahd_check_cmdcmpltqueues(struct ahd_softc *ahd)
 916 {
 917         u_int retval;
 918
 919         retval = 0;
 920         ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 921                         /*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo),
 922                         /*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD);
 923         if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag
 924           == ahd->qoutfifonext_valid_tag)
 925                 retval |= AHD_RUN_QOUTFIFO;
 926 #ifdef AHD_TARGET_MODE
 927         if ((ahd->flags & AHD_TARGETROLE) != 0
 928          && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) {
 929                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 930                                 ahd->shared_data_map.dmamap,
 931                                 ahd_targetcmd_offset(ahd, ahd->tqinfifofnext),
 932                                 /*len*/sizeof(struct target_cmd),
 933                                 BUS_DMASYNC_POSTREAD);
 934                 if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0)
 935                         retval |= AHD_RUN_TQINFIFO;
 936         }
 937 #endif
 938         return (retval);
 939 }
 940
 941 /*
 942  * Catch an interrupt from the adapter
 943  */
 944 int
 945 ahd_intr(struct ahd_softc *ahd)
 946 {
 947         u_int   intstat;
 948
 949         if ((ahd->pause & INTEN) == 0) {
 950                 /*
 951                  * Our interrupt is not enabled on the chip
 952                  * and may be disabled for re-entrancy reasons,
 953                  * so just return.  This is likely just a shared
 954                  * interrupt.
 955                  */
 956                 return (0);
 957         }
 958
 959         /*
 960          * Instead of directly reading the interrupt status register,
 961          * infer the cause of the interrupt by checking our in-core
 962          * completion queues.  This avoids a costly PCI bus read in
 963          * most cases.
 964          */
 965         if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0
 966          && (ahd_check_cmdcmpltqueues(ahd) != 0))
 967                 intstat = CMDCMPLT;
 968         else
 969                 intstat = ahd_inb(ahd, INTSTAT);
 970
 971         if ((intstat & INT_PEND) == 0)
 972                 return (0);
 973
 974         if (intstat & CMDCMPLT) {
 975                 ahd_outb(ahd, CLRINT, CLRCMDINT);
 976
 977                 /*
 978                  * Ensure that the chip sees that we've cleared
 979                  * this interrupt before we walk the output fifo.
 980                  * Otherwise, we may, due to posted bus writes,
 981                  * clear the interrupt after we finish the scan,
 982                  * and after the sequencer has added new entries
 983                  * and asserted the interrupt again.
 984                  */
 985                 if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 986                         if (ahd_is_paused(ahd)) {
 987                                 /*
 988                                  * Potentially lost SEQINT.
 989                                  * If SEQINTCODE is non-zero,
 990                                  * simulate the SEQINT.
 991                                  */
 992                                 if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT)
 993                                         intstat |= SEQINT;
 994                         }
 995                 } else {
 996                         ahd_flush_device_writes(ahd);
 997                 }
 998                 ahd_run_qoutfifo(ahd);
 999                 ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++;
1000                 ahd->cmdcmplt_total++;
1001 #ifdef AHD_TARGET_MODE
1002                 if ((ahd->flags & AHD_TARGETROLE) != 0)
1003                         ahd_run_tqinfifo(ahd, /*paused*/FALSE);
1004 #endif
1005         }
1006
1007         /*
1008          * Handle statuses that may invalidate our cached
1009          * copy of INTSTAT separately.
1010          */
1011         if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) {
1012                 /* Hot eject.  Do nothing */
1013         } else if (intstat & HWERRINT) {
1014                 ahd_handle_hwerrint(ahd);
1015         } else if ((intstat & (PCIINT|SPLTINT)) != 0) {
1016                 ahd->bus_intr(ahd);
1017         } else {
1018
1019                 if ((intstat & SEQINT) != 0)
1020                         ahd_handle_seqint(ahd, intstat);
1021
1022                 if ((intstat & SCSIINT) != 0)
1023                         ahd_handle_scsiint(ahd, intstat);
1024         }
1025         return (1);
1026 }
1027
1028 /******************************** Private Inlines *****************************/
1029 static inline void
1030 ahd_assert_atn(struct ahd_softc *ahd)
1031 {
1032         ahd_outb(ahd, SCSISIGO, ATNO);
1033 }
1034
1035 /*
1036  * Determine if the current connection has a packetized
1037  * agreement.  This does not necessarily mean that we
1038  * are currently in a packetized transfer.  We could
1039  * just as easily be sending or receiving a message.
1040  */
1041 static int
1042 ahd_currently_packetized(struct ahd_softc *ahd)
1043 {
1044         ahd_mode_state   saved_modes;
1045         int              packetized;
1046
1047         saved_modes = ahd_save_modes(ahd);
1048         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
1049                 /*
1050                  * The packetized bit refers to the last
1051                  * connection, not the current one.  Check
1052                  * for non-zero LQISTATE instead.
1053                  */
1054                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1055                 packetized = ahd_inb(ahd, LQISTATE) != 0;
1056         } else {
1057                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1058                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
1059         }
1060         ahd_restore_modes(ahd, saved_modes);
1061         return (packetized);
1062 }
1063
1064 static inline int
1065 ahd_set_active_fifo(struct ahd_softc *ahd)
1066 {
1067         u_int active_fifo;
1068
1069         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1070         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
1071         switch (active_fifo) {
1072         case 0:
1073         case 1:
1074                 ahd_set_modes(ahd, active_fifo, active_fifo);
1075                 return (1);
1076         default:
1077                 return (0);
1078         }
1079 }
1080
1081 static inline void
1082 ahd_unbusy_tcl(struct ahd_softc *ahd, u_int tcl)
1083 {
1084         ahd_busy_tcl(ahd, tcl, SCB_LIST_NULL);
1085 }
1086
1087 /*
1088  * Determine whether the sequencer reported a residual
1089  * for this SCB/transaction.
1090  */
1091 static inline void
1092 ahd_update_residual(struct ahd_softc *ahd, struct scb *scb)
1093 {
1094         uint32_t sgptr;
1095
1096         sgptr = ahd_le32toh(scb->hscb->sgptr);
1097         if ((sgptr & SG_STATUS_VALID) != 0)
1098                 ahd_calc_residual(ahd, scb);
1099 }
1100
1101 static inline void
1102 ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb)
1103 {
1104         uint32_t sgptr;
1105
1106         sgptr = ahd_le32toh(scb->hscb->sgptr);
1107         if ((sgptr & SG_STATUS_VALID) != 0)
1108                 ahd_handle_scb_status(ahd, scb);
1109         else
1110                 ahd_done(ahd, scb);
1111 }
1112
1113
1114 /************************* Sequencer Execution Control ************************/
1115 /*
1116  * Restart the sequencer program from address zero
1117  */
1118 static void
1119 ahd_restart(struct ahd_softc *ahd)
1120 {
1121
1122         ahd_pause(ahd);
1123
1124         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1125
1126         /* No more pending messages */
1127         ahd_clear_msg_state(ahd);
1128         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
1129         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
1130         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
1131         ahd_outb(ahd, SEQINTCTL, 0);
1132         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
1133         ahd_outb(ahd, SEQ_FLAGS, 0);
1134         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
1135         ahd_outb(ahd, SAVED_LUN, 0xFF);
1136
1137         /*
1138          * Ensure that the sequencer's idea of TQINPOS
1139          * matches our own.  The sequencer increments TQINPOS
1140          * only after it sees a DMA complete and a reset could
1141          * occur before the increment leaving the kernel to believe
1142          * the command arrived but the sequencer to not.
1143          */
1144         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
1145
1146         /* Always allow reselection */
1147         ahd_outb(ahd, SCSISEQ1,
1148                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
1149         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1150
1151         /*
1152          * Clear any pending sequencer interrupt.  It is no
1153          * longer relevant since we're resetting the Program
1154          * Counter.
1155          */
1156         ahd_outb(ahd, CLRINT, CLRSEQINT);
1157
1158         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
1159         ahd_unpause(ahd);
1160 }
1161
1162 static void
1163 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
1164 {
1165         ahd_mode_state   saved_modes;
1166
1167 #ifdef AHD_DEBUG
1168         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
1169                 printk("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
1170 #endif
1171         saved_modes = ahd_save_modes(ahd);
1172         ahd_set_modes(ahd, fifo, fifo);
1173         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
1174         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1175                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
1176         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1177         ahd_outb(ahd, SG_STATE, 0);
1178         ahd_restore_modes(ahd, saved_modes);
1179 }
1180
1181 /************************* Input/Output Queues ********************************/
1182 /*
1183  * Flush and completed commands that are sitting in the command
1184  * complete queues down on the chip but have yet to be dma'ed back up.
1185  */
1186 static void
1187 ahd_flush_qoutfifo(struct ahd_softc *ahd)
1188 {
1189         struct          scb *scb;
1190         ahd_mode_state  saved_modes;
1191         u_int           saved_scbptr;
1192         u_int           ccscbctl;
1193         u_int           scbid;
1194         u_int           next_scbid;
1195
1196         saved_modes = ahd_save_modes(ahd);
1197
1198         /*
1199          * Flush the good status FIFO for completed packetized commands.
1200          */
1201         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1202         saved_scbptr = ahd_get_scbptr(ahd);
1203         while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
1204                 u_int fifo_mode;
1205                 u_int i;
1206
1207                 scbid = ahd_inw(ahd, GSFIFO);
1208                 scb = ahd_lookup_scb(ahd, scbid);
1209                 if (scb == NULL) {
1210                         printk("%s: Warning - GSFIFO SCB %d invalid\n",
1211                                ahd_name(ahd), scbid);
1212                         continue;
1213                 }
1214                 /*
1215                  * Determine if this transaction is still active in
1216                  * any FIFO.  If it is, we must flush that FIFO to
1217                  * the host before completing the  command.
1218                  */
1219                 fifo_mode = 0;
1220 rescan_fifos:
1221                 for (i = 0; i < 2; i++) {
1222                         /* Toggle to the other mode. */
1223                         fifo_mode ^= 1;
1224                         ahd_set_modes(ahd, fifo_mode, fifo_mode);
1225
1226                         if (ahd_scb_active_in_fifo(ahd, scb) == 0)
1227                                 continue;
1228
1229                         ahd_run_data_fifo(ahd, scb);
1230
1231                         /*
1232                          * Running this FIFO may cause a CFG4DATA for
1233                          * this same transaction to assert in the other
1234                          * FIFO or a new snapshot SAVEPTRS interrupt
1235                          * in this FIFO.  Even running a FIFO may not
1236                          * clear the transaction if we are still waiting
1237                          * for data to drain to the host. We must loop
1238                          * until the transaction is not active in either
1239                          * FIFO just to be sure.  Reset our loop counter
1240                          * so we will visit both FIFOs again before
1241                          * declaring this transaction finished.  We
1242                          * also delay a bit so that status has a chance
1243                          * to change before we look at this FIFO again.
1244                          */
1245                         ahd_delay(200);
1246                         goto rescan_fifos;
1247                 }
1248                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1249                 ahd_set_scbptr(ahd, scbid);
1250                 if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
1251                  && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
1252                   || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
1253                       & SG_LIST_NULL) != 0)) {
1254                         u_int comp_head;
1255
1256                         /*
1257                          * The transfer completed with a residual.
1258                          * Place this SCB on the complete DMA list
1259                          * so that we update our in-core copy of the
1260                          * SCB before completing the command.
1261                          */
1262                         ahd_outb(ahd, SCB_SCSI_STATUS, 0);
1263                         ahd_outb(ahd, SCB_SGPTR,
1264                                  ahd_inb_scbram(ahd, SCB_SGPTR)
1265                                  | SG_STATUS_VALID);
1266                         ahd_outw(ahd, SCB_TAG, scbid);
1267                         ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
1268                         comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1269                         if (SCBID_IS_NULL(comp_head)) {
1270                                 ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
1271                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1272                         } else {
1273                                 u_int tail;
1274
1275                                 tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
1276                                 ahd_set_scbptr(ahd, tail);
1277                                 ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
1278                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1279                                 ahd_set_scbptr(ahd, scbid);
1280                         }
1281                 } else
1282                         ahd_complete_scb(ahd, scb);
1283         }
1284         ahd_set_scbptr(ahd, saved_scbptr);
1285
1286         /*
1287          * Setup for command channel portion of flush.
1288          */
1289         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1290
1291         /*
1292          * Wait for any inprogress DMA to complete and clear DMA state
1293          * if this is for an SCB in the qinfifo.
1294          */
1295         while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
1296
1297                 if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
1298                         if ((ccscbctl & ARRDONE) != 0)
1299                                 break;
1300                 } else if ((ccscbctl & CCSCBDONE) != 0)
1301                         break;
1302                 ahd_delay(200);
1303         }
1304         /*
1305          * We leave the sequencer to cleanup in the case of DMA's to
1306          * update the qoutfifo.  In all other cases (DMA's to the
1307          * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
1308          * we disable the DMA engine so that the sequencer will not
1309          * attempt to handle the DMA completion.
1310          */
1311         if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
1312                 ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
1313
1314         /*
1315          * Complete any SCBs that just finished
1316          * being DMA'ed into the qoutfifo.
1317          */
1318         ahd_run_qoutfifo(ahd);
1319
1320         saved_scbptr = ahd_get_scbptr(ahd);
1321         /*
1322          * Manually update/complete any completed SCBs that are waiting to be
1323          * DMA'ed back up to the host.
1324          */
1325         scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1326         while (!SCBID_IS_NULL(scbid)) {
1327                 uint8_t *hscb_ptr;
1328                 u_int    i;
1329
1330                 ahd_set_scbptr(ahd, scbid);
1331                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1332                 scb = ahd_lookup_scb(ahd, scbid);
1333                 if (scb == NULL) {
1334                         printk("%s: Warning - DMA-up and complete "
1335                                "SCB %d invalid\n", ahd_name(ahd), scbid);
1336                         continue;
1337                 }
1338                 hscb_ptr = (uint8_t *)scb->hscb;
1339                 for (i = 0; i < sizeof(struct hardware_scb); i++)
1340                         *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
1341
1342                 ahd_complete_scb(ahd, scb);
1343                 scbid = next_scbid;
1344         }
1345         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
1346         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
1347
1348         scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
1349         while (!SCBID_IS_NULL(scbid)) {
1350
1351                 ahd_set_scbptr(ahd, scbid);
1352                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1353                 scb = ahd_lookup_scb(ahd, scbid);
1354                 if (scb == NULL) {
1355                         printk("%s: Warning - Complete Qfrz SCB %d invalid\n",
1356                                ahd_name(ahd), scbid);
1357                         continue;
1358                 }
1359
1360                 ahd_complete_scb(ahd, scb);
1361                 scbid = next_scbid;
1362         }
1363         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
1364
1365         scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
1366         while (!SCBID_IS_NULL(scbid)) {
1367
1368                 ahd_set_scbptr(ahd, scbid);
1369                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1370                 scb = ahd_lookup_scb(ahd, scbid);
1371                 if (scb == NULL) {
1372                         printk("%s: Warning - Complete SCB %d invalid\n",
1373                                ahd_name(ahd), scbid);
1374                         continue;
1375                 }
1376
1377                 ahd_complete_scb(ahd, scb);
1378                 scbid = next_scbid;
1379         }
1380         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
1381
1382         /*
1383          * Restore state.
1384          */
1385         ahd_set_scbptr(ahd, saved_scbptr);
1386         ahd_restore_modes(ahd, saved_modes);
1387         ahd->flags |= AHD_UPDATE_PEND_CMDS;
1388 }
1389
1390 /*
1391  * Determine if an SCB for a packetized transaction
1392  * is active in a FIFO.
1393  */
1394 static int
1395 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
1396 {
1397
1398         /*
1399          * The FIFO is only active for our transaction if
1400          * the SCBPTR matches the SCB's ID and the firmware
1401          * has installed a handler for the FIFO or we have
1402          * a pending SAVEPTRS or CFG4DATA interrupt.
1403          */
1404         if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
1405          || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
1406           && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
1407                 return (0);
1408
1409         return (1);
1410 }
1411
1412 /*
1413  * Run a data fifo to completion for a transaction we know
1414  * has completed across the SCSI bus (good status has been
1415  * received).  We are already set to the correct FIFO mode
1416  * on entry to this routine.
1417  *
1418  * This function attempts to operate exactly as the firmware
1419  * would when running this FIFO.  Care must be taken to update
1420  * this routine any time the firmware's FIFO algorithm is
1421  * changed.
1422  */
1423 static void
1424 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
1425 {
1426         u_int seqintsrc;
1427
1428         seqintsrc = ahd_inb(ahd, SEQINTSRC);
1429         if ((seqintsrc & CFG4DATA) != 0) {
1430                 uint32_t datacnt;
1431                 uint32_t sgptr;
1432
1433                 /*
1434                  * Clear full residual flag.
1435                  */
1436                 sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
1437                 ahd_outb(ahd, SCB_SGPTR, sgptr);
1438
1439                 /*
1440                  * Load datacnt and address.
1441                  */
1442                 datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
1443                 if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
1444                         sgptr |= LAST_SEG;
1445                         ahd_outb(ahd, SG_STATE, 0);
1446                 } else
1447                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1448                 ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
1449                 ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
1450                 ahd_outb(ahd, SG_CACHE_PRE, sgptr);
1451                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1452
1453                 /*
1454                  * Initialize Residual Fields.
1455                  */
1456                 ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
1457                 ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
1458
1459                 /*
1460                  * Mark the SCB as having a FIFO in use.
1461                  */
1462                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1463                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
1464
1465                 /*
1466                  * Install a "fake" handler for this FIFO.
1467                  */
1468                 ahd_outw(ahd, LONGJMP_ADDR, 0);
1469
1470                 /*
1471                  * Notify the hardware that we have satisfied
1472                  * this sequencer interrupt.
1473                  */
1474                 ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
1475         } else if ((seqintsrc & SAVEPTRS) != 0) {
1476                 uint32_t sgptr;
1477                 uint32_t resid;
1478
1479                 if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
1480                         /*
1481                          * Snapshot Save Pointers.  All that
1482                          * is necessary to clear the snapshot
1483                          * is a CLRCHN.
1484                          */
1485                         goto clrchn;
1486                 }
1487
1488                 /*
1489                  * Disable S/G fetch so the DMA engine
1490                  * is available to future users.
1491                  */
1492                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1493                         ahd_outb(ahd, CCSGCTL, 0);
1494                 ahd_outb(ahd, SG_STATE, 0);
1495
1496                 /*
1497                  * Flush the data FIFO.  Strickly only
1498                  * necessary for Rev A parts.
1499                  */
1500                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
1501
1502                 /*
1503                  * Calculate residual.
1504                  */
1505                 sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1506                 resid = ahd_inl(ahd, SHCNT);
1507                 resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
1508                 ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
1509                 if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
1510                         /*
1511                          * Must back up to the correct S/G element.
1512                          * Typically this just means resetting our
1513                          * low byte to the offset in the SG_CACHE,
1514                          * but if we wrapped, we have to correct
1515                          * the other bytes of the sgptr too.
1516                          */
1517                         if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
1518                          && (sgptr & 0x80) == 0)
1519                                 sgptr -= 0x100;
1520                         sgptr &= ~0xFF;
1521                         sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
1522                                & SG_ADDR_MASK;
1523                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1524                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
1525                 } else if ((resid & AHD_SG_LEN_MASK) == 0) {
1526                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
1527                                  sgptr | SG_LIST_NULL);
1528                 }
1529                 /*
1530                  * Save Pointers.
1531                  */
1532                 ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
1533                 ahd_outl(ahd, SCB_DATACNT, resid);
1534                 ahd_outl(ahd, SCB_SGPTR, sgptr);
1535                 ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
1536                 ahd_outb(ahd, SEQIMODE,
1537                          ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
1538                 /*
1539                  * If the data is to the SCSI bus, we are
1540                  * done, otherwise wait for FIFOEMP.
1541                  */
1542                 if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
1543                         goto clrchn;
1544         } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
1545                 uint32_t sgptr;
1546                 uint64_t data_addr;
1547                 uint32_t data_len;
1548                 u_int    dfcntrl;
1549
1550                 /*
1551                  * Disable S/G fetch so the DMA engine
1552                  * is available to future users.  We won't
1553                  * be using the DMA engine to load segments.
1554                  */
1555                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
1556                         ahd_outb(ahd, CCSGCTL, 0);
1557                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1558                 }
1559
1560                 /*
1561                  * Wait for the DMA engine to notice that the
1562                  * host transfer is enabled and that there is
1563                  * space in the S/G FIFO for new segments before
1564                  * loading more segments.
1565                  */
1566                 if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
1567                  && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
1568
1569                         /*
1570                          * Determine the offset of the next S/G
1571                          * element to load.
1572                          */
1573                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1574                         sgptr &= SG_PTR_MASK;
1575                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
1576                                 struct ahd_dma64_seg *sg;
1577
1578                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1579                                 data_addr = sg->addr;
1580                                 data_len = sg->len;
1581                                 sgptr += sizeof(*sg);
1582                         } else {
1583                                 struct  ahd_dma_seg *sg;
1584
1585                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1586                                 data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
1587                                 data_addr <<= 8;
1588                                 data_addr |= sg->addr;
1589                                 data_len = sg->len;
1590                                 sgptr += sizeof(*sg);
1591                         }
1592
1593                         /*
1594                          * Update residual information.
1595                          */
1596                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
1597                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1598
1599                         /*
1600                          * Load the S/G.
1601                          */
1602                         if (data_len & AHD_DMA_LAST_SEG) {
1603                                 sgptr |= LAST_SEG;
1604                                 ahd_outb(ahd, SG_STATE, 0);
1605                         }
1606                         ahd_outq(ahd, HADDR, data_addr);
1607                         ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
1608                         ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
1609
1610                         /*
1611                          * Advertise the segment to the hardware.
1612                          */
1613                         dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
1614                         if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
1615                                 /*
1616                                  * Use SCSIENWRDIS so that SCSIEN
1617                                  * is never modified by this
1618                                  * operation.
1619                                  */
1620                                 dfcntrl |= SCSIENWRDIS;
1621                         }
1622                         ahd_outb(ahd, DFCNTRL, dfcntrl);
1623                 }
1624         } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
1625
1626                 /*
1627                  * Transfer completed to the end of SG list
1628                  * and has flushed to the host.
1629                  */
1630                 ahd_outb(ahd, SCB_SGPTR,
1631                          ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
1632                 goto clrchn;
1633         } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
1634 clrchn:
1635                 /*
1636                  * Clear any handler for this FIFO, decrement
1637                  * the FIFO use count for the SCB, and release
1638                  * the FIFO.
1639                  */
1640                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1641                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1642                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
1643                 ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
1644         }
1645 }
1646
1647 /*
1648  * Look for entries in the QoutFIFO that have completed.
1649  * The valid_tag completion field indicates the validity
1650  * of the entry - the valid value toggles each time through
1651  * the queue. We use the sg_status field in the completion
1652  * entry to avoid referencing the hscb if the completion
1653  * occurred with no errors and no residual.  sg_status is
1654  * a copy of the first byte (little endian) of the sgptr
1655  * hscb field.
1656  */
1657 static void
1658 ahd_run_qoutfifo(struct ahd_softc *ahd)
1659 {
1660         struct ahd_completion *completion;
1661         struct scb *scb;
1662         u_int  scb_index;
1663
1664         if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
1665                 panic("ahd_run_qoutfifo recursion");
1666         ahd->flags |= AHD_RUNNING_QOUTFIFO;
1667         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
1668         for (;;) {
1669                 completion = &ahd->qoutfifo[ahd->qoutfifonext];
1670
1671                 if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
1672                         break;
1673
1674                 scb_index = ahd_le16toh(completion->tag);
1675                 scb = ahd_lookup_scb(ahd, scb_index);
1676                 if (scb == NULL) {
1677                         printk("%s: WARNING no command for scb %d "
1678                                "(cmdcmplt)\nQOUTPOS = %d\n",
1679                                ahd_name(ahd), scb_index,
1680                                ahd->qoutfifonext);
1681                         ahd_dump_card_state(ahd);
1682                 } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
1683                         ahd_handle_scb_status(ahd, scb);
1684                 } else {
1685                         ahd_done(ahd, scb);
1686                 }
1687
1688                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
1689                 if (ahd->qoutfifonext == 0)
1690                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
1691         }
1692         ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
1693 }
1694
1695 /************************* Interrupt Handling *********************************/
1696 static void
1697 ahd_handle_hwerrint(struct ahd_softc *ahd)
1698 {
1699         /*
1700          * Some catastrophic hardware error has occurred.
1701          * Print it for the user and disable the controller.
1702          */
1703         int i;
1704         int error;
1705
1706         error = ahd_inb(ahd, ERROR);
1707         for (i = 0; i < num_errors; i++) {
1708                 if ((error & ahd_hard_errors[i].errno) != 0)
1709                         printk("%s: hwerrint, %s\n",
1710                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
1711         }
1712
1713         ahd_dump_card_state(ahd);
1714         panic("BRKADRINT");
1715
1716         /* Tell everyone that this HBA is no longer available */
1717         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
1718                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
1719                        CAM_NO_HBA);
1720
1721         /* Tell the system that this controller has gone away. */
1722         ahd_free(ahd);
1723 }
1724
1725 #ifdef AHD_DEBUG
1726 static void
1727 ahd_dump_sglist(struct scb *scb)
1728 {
1729         int i;
1730
1731         if (scb->sg_count > 0) {
1732                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
1733                         struct ahd_dma64_seg *sg_list;
1734
1735                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
1736                         for (i = 0; i < scb->sg_count; i++) {
1737                                 uint64_t addr;
1738
1739                                 addr = ahd_le64toh(sg_list[i].addr);
1740                                 printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1741                                        i,
1742                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
1743                                        (uint32_t)(addr & 0xFFFFFFFF),
1744                                        sg_list[i].len & AHD_SG_LEN_MASK,
1745                                        (sg_list[i].len & AHD_DMA_LAST_SEG)
1746                                      ? " Last" : "");
1747                         }
1748                 } else {
1749                         struct ahd_dma_seg *sg_list;
1750
1751                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
1752                         for (i = 0; i < scb->sg_count; i++) {
1753                                 uint32_t len;
1754
1755                                 len = ahd_le32toh(sg_list[i].len);
1756                                 printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1757                                        i,
1758                                        (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
1759                                        ahd_le32toh(sg_list[i].addr),
1760                                        len & AHD_SG_LEN_MASK,
1761                                        len & AHD_DMA_LAST_SEG ? " Last" : "");
1762                         }
1763                 }
1764         }
1765 }
1766 #endif  /*  AHD_DEBUG  */
1767
1768 static void
1769 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
1770 {
1771         u_int seqintcode;
1772
1773         /*
1774          * Save the sequencer interrupt code and clear the SEQINT
1775          * bit. We will unpause the sequencer, if appropriate,
1776          * after servicing the request.
1777          */
1778         seqintcode = ahd_inb(ahd, SEQINTCODE);
1779         ahd_outb(ahd, CLRINT, CLRSEQINT);
1780         if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
1781                 /*
1782                  * Unpause the sequencer and let it clear
1783                  * SEQINT by writing NO_SEQINT to it.  This
1784                  * will cause the sequencer to be paused again,
1785                  * which is the expected state of this routine.
1786                  */
1787                 ahd_unpause(ahd);
1788                 while (!ahd_is_paused(ahd))
1789                         ;
1790                 ahd_outb(ahd, CLRINT, CLRSEQINT);
1791         }
1792         ahd_update_modes(ahd);
1793 #ifdef AHD_DEBUG
1794         if ((ahd_debug & AHD_SHOW_MISC) != 0)
1795                 printk("%s: Handle Seqint Called for code %d\n",
1796                        ahd_name(ahd), seqintcode);
1797 #endif
1798         switch (seqintcode) {
1799         case ENTERING_NONPACK:
1800         {
1801                 struct  scb *scb;
1802                 u_int   scbid;
1803
1804                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1805                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1806                 scbid = ahd_get_scbptr(ahd);
1807                 scb = ahd_lookup_scb(ahd, scbid);
1808                 if (scb == NULL) {
1809                         /*
1810                          * Somehow need to know if this
1811                          * is from a selection or reselection.
1812                          * From that, we can determine target
1813                          * ID so we at least have an I_T nexus.
1814                          */
1815                 } else {
1816                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1817                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
1818                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
1819                 }
1820                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
1821                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
1822                         /*
1823                          * Phase change after read stream with
1824                          * CRC error with P0 asserted on last
1825                          * packet.
1826                          */
1827 #ifdef AHD_DEBUG
1828                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1829                                 printk("%s: Assuming LQIPHASE_NLQ with "
1830                                        "P0 assertion\n", ahd_name(ahd));
1831 #endif
1832                 }
1833 #ifdef AHD_DEBUG
1834                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1835                         printk("%s: Entering NONPACK\n", ahd_name(ahd));
1836 #endif
1837                 break;
1838         }
1839         case INVALID_SEQINT:
1840                 printk("%s: Invalid Sequencer interrupt occurred, "
1841                        "resetting channel.\n",
1842                        ahd_name(ahd));
1843 #ifdef AHD_DEBUG
1844                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1845                         ahd_dump_card_state(ahd);
1846 #endif
1847                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1848                 break;
1849         case STATUS_OVERRUN:
1850         {
1851                 struct  scb *scb;
1852                 u_int   scbid;
1853
1854                 scbid = ahd_get_scbptr(ahd);
1855                 scb = ahd_lookup_scb(ahd, scbid);
1856                 if (scb != NULL)
1857                         ahd_print_path(ahd, scb);
1858                 else
1859                         printk("%s: ", ahd_name(ahd));
1860                 printk("SCB %d Packetized Status Overrun", scbid);
1861                 ahd_dump_card_state(ahd);
1862                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1863                 break;
1864         }
1865         case CFG4ISTAT_INTR:
1866         {
1867                 struct  scb *scb;
1868                 u_int   scbid;
1869
1870                 scbid = ahd_get_scbptr(ahd);
1871                 scb = ahd_lookup_scb(ahd, scbid);
1872                 if (scb == NULL) {
1873                         ahd_dump_card_state(ahd);
1874                         printk("CFG4ISTAT: Free SCB %d referenced", scbid);
1875                         panic("For safety");
1876                 }
1877                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
1878                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1879                 ahd_outb(ahd, HCNT + 2, 0);
1880                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1881                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1882                 break;
1883         }
1884         case ILLEGAL_PHASE:
1885         {
1886                 u_int bus_phase;
1887
1888                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1889                 printk("%s: ILLEGAL_PHASE 0x%x\n",
1890                        ahd_name(ahd), bus_phase);
1891
1892                 switch (bus_phase) {
1893                 case P_DATAOUT:
1894                 case P_DATAIN:
1895                 case P_DATAOUT_DT:
1896                 case P_DATAIN_DT:
1897                 case P_MESGOUT:
1898                 case P_STATUS:
1899                 case P_MESGIN:
1900                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1901                         printk("%s: Issued Bus Reset.\n", ahd_name(ahd));
1902                         break;
1903                 case P_COMMAND:
1904                 {
1905                         struct  ahd_devinfo devinfo;
1906                         struct  scb *scb;
1907                         u_int   scbid;
1908
1909                         /*
1910                          * If a target takes us into the command phase
1911                          * assume that it has been externally reset and
1912                          * has thus lost our previous packetized negotiation
1913                          * agreement.  Since we have not sent an identify
1914                          * message and may not have fully qualified the
1915                          * connection, we change our command to TUR, assert
1916                          * ATN and ABORT the task when we go to message in
1917                          * phase.  The OSM will see the REQUEUE_REQUEST
1918                          * status and retry the command.
1919                          */
1920                         scbid = ahd_get_scbptr(ahd);
1921                         scb = ahd_lookup_scb(ahd, scbid);
1922                         if (scb == NULL) {
1923                                 printk("Invalid phase with no valid SCB.  "
1924                                        "Resetting bus.\n");
1925                                 ahd_reset_channel(ahd, 'A',
1926                                                   /*Initiate Reset*/TRUE);
1927                                 break;
1928                         }
1929                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1930                                             SCB_GET_TARGET(ahd, scb),
1931                                             SCB_GET_LUN(scb),
1932                                             SCB_GET_CHANNEL(ahd, scb),
1933                                             ROLE_INITIATOR);
1934                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1935                                       AHD_TRANS_ACTIVE, /*paused*/TRUE);
1936                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1937                                          /*offset*/0, /*ppr_options*/0,
1938                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
1939                         /* Hand-craft TUR command */
1940                         ahd_outb(ahd, SCB_CDB_STORE, 0);
1941                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1942                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1943                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1944                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1945                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1946                         ahd_outb(ahd, SCB_CDB_LEN, 6);
1947                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1948                         scb->hscb->control |= MK_MESSAGE;
1949                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1950                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1951                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1952                         /*
1953                          * The lun is 0, regardless of the SCB's lun
1954                          * as we have not sent an identify message.
1955                          */
1956                         ahd_outb(ahd, SAVED_LUN, 0);
1957                         ahd_outb(ahd, SEQ_FLAGS, 0);
1958                         ahd_assert_atn(ahd);
1959                         scb->flags &= ~SCB_PACKETIZED;
1960                         scb->flags |= SCB_ABORT|SCB_EXTERNAL_RESET;
1961                         ahd_freeze_devq(ahd, scb);
1962                         ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1963                         ahd_freeze_scb(scb);
1964
1965                         /* Notify XPT */
1966                         ahd_send_async(ahd, devinfo.channel, devinfo.target,
1967                                        CAM_LUN_WILDCARD, AC_SENT_BDR);
1968
1969                         /*
1970                          * Allow the sequencer to continue with
1971                          * non-pack processing.
1972                          */
1973                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1974                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1975                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1976                                 ahd_outb(ahd, CLRLQOINT1, 0);
1977                         }
1978 #ifdef AHD_DEBUG
1979                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1980                                 ahd_print_path(ahd, scb);
1981                                 printk("Unexpected command phase from "
1982                                        "packetized target\n");
1983                         }
1984 #endif
1985                         break;
1986                 }
1987                 }
1988                 break;
1989         }
1990         case CFG4OVERRUN:
1991         {
1992                 struct  scb *scb;
1993                 u_int   scb_index;
1994
1995 #ifdef AHD_DEBUG
1996                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1997                         printk("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1998                                ahd_inb(ahd, MODE_PTR));
1999                 }
2000 #endif
2001                 scb_index = ahd_get_scbptr(ahd);
2002                 scb = ahd_lookup_scb(ahd, scb_index);
2003                 if (scb == NULL) {
2004                         /*
2005                          * Attempt to transfer to an SCB that is
2006                          * not outstanding.
2007                          */
2008                         ahd_assert_atn(ahd);
2009                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2010                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
2011                         ahd->msgout_len = 1;
2012                         ahd->msgout_index = 0;
2013                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2014                         /*
2015                          * Clear status received flag to prevent any
2016                          * attempt to complete this bogus SCB.
2017                          */
2018                         ahd_outb(ahd, SCB_CONTROL,
2019                                  ahd_inb_scbram(ahd, SCB_CONTROL)
2020                                  & ~STATUS_RCVD);
2021                 }
2022                 break;
2023         }
2024         case DUMP_CARD_STATE:
2025         {
2026                 ahd_dump_card_state(ahd);
2027                 break;
2028         }
2029         case PDATA_REINIT:
2030         {
2031 #ifdef AHD_DEBUG
2032                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2033                         printk("%s: PDATA_REINIT - DFCNTRL = 0x%x "
2034                                "SG_CACHE_SHADOW = 0x%x\n",
2035                                ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
2036                                ahd_inb(ahd, SG_CACHE_SHADOW));
2037                 }
2038 #endif
2039                 ahd_reinitialize_dataptrs(ahd);
2040                 break;
2041         }
2042         case HOST_MSG_LOOP:
2043         {
2044                 struct ahd_devinfo devinfo;
2045
2046                 /*
2047                  * The sequencer has encountered a message phase
2048                  * that requires host assistance for completion.
2049                  * While handling the message phase(s), we will be
2050                  * notified by the sequencer after each byte is
2051                  * transferred so we can track bus phase changes.
2052                  *
2053                  * If this is the first time we've seen a HOST_MSG_LOOP
2054                  * interrupt, initialize the state of the host message
2055                  * loop.
2056                  */
2057                 ahd_fetch_devinfo(ahd, &devinfo);
2058                 if (ahd->msg_type == MSG_TYPE_NONE) {
2059                         struct scb *scb;
2060                         u_int scb_index;
2061                         u_int bus_phase;
2062
2063                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2064                         if (bus_phase != P_MESGIN
2065                          && bus_phase != P_MESGOUT) {
2066                                 printk("ahd_intr: HOST_MSG_LOOP bad "
2067                                        "phase 0x%x\n", bus_phase);
2068                                 /*
2069                                  * Probably transitioned to bus free before
2070                                  * we got here.  Just punt the message.
2071                                  */
2072                                 ahd_dump_card_state(ahd);
2073                                 ahd_clear_intstat(ahd);
2074                                 ahd_restart(ahd);
2075                                 return;
2076                         }
2077
2078                         scb_index = ahd_get_scbptr(ahd);
2079                         scb = ahd_lookup_scb(ahd, scb_index);
2080                         if (devinfo.role == ROLE_INITIATOR) {
2081                                 if (bus_phase == P_MESGOUT)
2082                                         ahd_setup_initiator_msgout(ahd,
2083                                                                    &devinfo,
2084                                                                    scb);
2085                                 else {
2086                                         ahd->msg_type =
2087                                             MSG_TYPE_INITIATOR_MSGIN;
2088                                         ahd->msgin_index = 0;
2089                                 }
2090                         }
2091 #ifdef AHD_TARGET_MODE
2092                         else {
2093                                 if (bus_phase == P_MESGOUT) {
2094                                         ahd->msg_type =
2095                                             MSG_TYPE_TARGET_MSGOUT;
2096                                         ahd->msgin_index = 0;
2097                                 }
2098                                 else
2099                                         ahd_setup_target_msgin(ahd,
2100                                                                &devinfo,
2101                                                                scb);
2102                         }
2103 #endif
2104                 }
2105
2106                 ahd_handle_message_phase(ahd);
2107                 break;
2108         }
2109         case NO_MATCH:
2110         {
2111                 /* Ensure we don't leave the selection hardware on */
2112                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
2113                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2114
2115                 printk("%s:%c:%d: no active SCB for reconnecting "
2116                        "target - issuing BUS DEVICE RESET\n",
2117                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
2118                 printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
2119                        "REG0 == 0x%x ACCUM = 0x%x\n",
2120                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
2121                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
2122                 printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
2123                        "SINDEX == 0x%x\n",
2124                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
2125                        ahd_find_busy_tcl(ahd,
2126                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
2127                                                    ahd_inb(ahd, SAVED_LUN))),
2128                        ahd_inw(ahd, SINDEX));
2129                 printk("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
2130                        "SCB_CONTROL == 0x%x\n",
2131                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
2132                        ahd_inb_scbram(ahd, SCB_LUN),
2133                        ahd_inb_scbram(ahd, SCB_CONTROL));
2134                 printk("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
2135                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
2136                 printk("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
2137                 printk("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
2138                 ahd_dump_card_state(ahd);
2139                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
2140                 ahd->msgout_len = 1;
2141                 ahd->msgout_index = 0;
2142                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2143                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
2144                 ahd_assert_atn(ahd);
2145                 break;
2146         }
2147         case PROTO_VIOLATION:
2148         {
2149                 ahd_handle_proto_violation(ahd);
2150                 break;
2151         }
2152         case IGN_WIDE_RES:
2153         {
2154                 struct ahd_devinfo devinfo;
2155
2156                 ahd_fetch_devinfo(ahd, &devinfo);
2157                 ahd_handle_ign_wide_residue(ahd, &devinfo);
2158                 break;
2159         }
2160         case BAD_PHASE:
2161         {
2162                 u_int lastphase;
2163
2164                 lastphase = ahd_inb(ahd, LASTPHASE);
2165                 printk("%s:%c:%d: unknown scsi bus phase %x, "
2166                        "lastphase = 0x%x.  Attempting to continue\n",
2167                        ahd_name(ahd), 'A',
2168                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2169                        lastphase, ahd_inb(ahd, SCSISIGI));
2170                 break;
2171         }
2172         case MISSED_BUSFREE:
2173         {
2174                 u_int lastphase;
2175
2176                 lastphase = ahd_inb(ahd, LASTPHASE);
2177                 printk("%s:%c:%d: Missed busfree. "
2178                        "Lastphase = 0x%x, Curphase = 0x%x\n",
2179                        ahd_name(ahd), 'A',
2180                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2181                        lastphase, ahd_inb(ahd, SCSISIGI));
2182                 ahd_restart(ahd);
2183                 return;
2184         }
2185         case DATA_OVERRUN:
2186         {
2187                 /*
2188                  * When the sequencer detects an overrun, it
2189                  * places the controller in "BITBUCKET" mode
2190                  * and allows the target to complete its transfer.
2191                  * Unfortunately, none of the counters get updated
2192                  * when the controller is in this mode, so we have
2193                  * no way of knowing how large the overrun was.
2194                  */
2195                 struct  scb *scb;
2196                 u_int   scbindex;
2197 #ifdef AHD_DEBUG
2198                 u_int   lastphase;
2199 #endif
2200
2201                 scbindex = ahd_get_scbptr(ahd);
2202                 scb = ahd_lookup_scb(ahd, scbindex);
2203 #ifdef AHD_DEBUG
2204                 lastphase = ahd_inb(ahd, LASTPHASE);
2205                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2206                         ahd_print_path(ahd, scb);
2207                         printk("data overrun detected %s.  Tag == 0x%x.\n",
2208                                ahd_lookup_phase_entry(lastphase)->phasemsg,
2209                                SCB_GET_TAG(scb));
2210                         ahd_print_path(ahd, scb);
2211                         printk("%s seen Data Phase.  Length = %ld.  "
2212                                "NumSGs = %d.\n",
2213                                ahd_inb(ahd, SEQ_FLAGS) & DPHASE
2214                                ? "Have" : "Haven't",
2215                                ahd_get_transfer_length(scb), scb->sg_count);
2216                         ahd_dump_sglist(scb);
2217                 }
2218 #endif
2219
2220                 /*
2221                  * Set this and it will take effect when the
2222                  * target does a command complete.
2223                  */
2224                 ahd_freeze_devq(ahd, scb);
2225                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
2226                 ahd_freeze_scb(scb);
2227                 break;
2228         }
2229         case MKMSG_FAILED:
2230         {
2231                 struct ahd_devinfo devinfo;
2232                 struct scb *scb;
2233                 u_int scbid;
2234
2235                 ahd_fetch_devinfo(ahd, &devinfo);
2236                 printk("%s:%c:%d:%d: Attempt to issue message failed\n",
2237                        ahd_name(ahd), devinfo.channel, devinfo.target,
2238                        devinfo.lun);
2239                 scbid = ahd_get_scbptr(ahd);
2240                 scb = ahd_lookup_scb(ahd, scbid);
2241                 if (scb != NULL
2242                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
2243                         /*
2244                          * Ensure that we didn't put a second instance of this
2245                          * SCB into the QINFIFO.
2246                          */
2247                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2248                                            SCB_GET_CHANNEL(ahd, scb),
2249                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2250                                            ROLE_INITIATOR, /*status*/0,
2251                                            SEARCH_REMOVE);
2252                 ahd_outb(ahd, SCB_CONTROL,
2253                          ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
2254                 break;
2255         }
2256         case TASKMGMT_FUNC_COMPLETE:
2257         {
2258                 u_int   scbid;
2259                 struct  scb *scb;
2260
2261                 scbid = ahd_get_scbptr(ahd);
2262                 scb = ahd_lookup_scb(ahd, scbid);
2263                 if (scb != NULL) {
2264                         u_int      lun;
2265                         u_int      tag;
2266                         cam_status error;
2267
2268                         ahd_print_path(ahd, scb);
2269                         printk("Task Management Func 0x%x Complete\n",
2270                                scb->hscb->task_management);
2271                         lun = CAM_LUN_WILDCARD;
2272                         tag = SCB_LIST_NULL;
2273
2274                         switch (scb->hscb->task_management) {
2275                         case SIU_TASKMGMT_ABORT_TASK:
2276                                 tag = SCB_GET_TAG(scb);
2277                                 fallthrough;
2278                         case SIU_TASKMGMT_ABORT_TASK_SET:
2279                         case SIU_TASKMGMT_CLEAR_TASK_SET:
2280                                 lun = scb->hscb->lun;
2281                                 error = CAM_REQ_ABORTED;
2282                                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2283                                                'A', lun, tag, ROLE_INITIATOR,
2284                                                error);
2285                                 break;
2286                         case SIU_TASKMGMT_LUN_RESET:
2287                                 lun = scb->hscb->lun;
2288                                 fallthrough;
2289                         case SIU_TASKMGMT_TARGET_RESET:
2290                         {
2291                                 struct ahd_devinfo devinfo;
2292
2293                                 ahd_scb_devinfo(ahd, &devinfo, scb);
2294                                 error = CAM_BDR_SENT;
2295                                 ahd_handle_devreset(ahd, &devinfo, lun,
2296                                                     CAM_BDR_SENT,
2297                                                     lun != CAM_LUN_WILDCARD
2298                                                     ? "Lun Reset"
2299                                                     : "Target Reset",
2300                                                     /*verbose_level*/0);
2301                                 break;
2302                         }
2303                         default:
2304                                 panic("Unexpected TaskMgmt Func\n");
2305                                 break;
2306                         }
2307                 }
2308                 break;
2309         }
2310         case TASKMGMT_CMD_CMPLT_OKAY:
2311         {
2312                 u_int   scbid;
2313                 struct  scb *scb;
2314
2315                 /*
2316                  * An ABORT TASK TMF failed to be delivered before
2317                  * the targeted command completed normally.
2318                  */
2319                 scbid = ahd_get_scbptr(ahd);
2320                 scb = ahd_lookup_scb(ahd, scbid);
2321                 if (scb != NULL) {
2322                         /*
2323                          * Remove the second instance of this SCB from
2324                          * the QINFIFO if it is still there.
2325                          */
2326                         ahd_print_path(ahd, scb);
2327                         printk("SCB completes before TMF\n");
2328                         /*
2329                          * Handle losing the race.  Wait until any
2330                          * current selection completes.  We will then
2331                          * set the TMF back to zero in this SCB so that
2332                          * the sequencer doesn't bother to issue another
2333                          * sequencer interrupt for its completion.
2334                          */
2335                         while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
2336                             && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
2337                             && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
2338                                 ;
2339                         ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
2340                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2341                                            SCB_GET_CHANNEL(ahd, scb),
2342                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2343                                            ROLE_INITIATOR, /*status*/0,
2344                                            SEARCH_REMOVE);
2345                 }
2346                 break;
2347         }
2348         case TRACEPOINT0:
2349         case TRACEPOINT1:
2350         case TRACEPOINT2:
2351         case TRACEPOINT3:
2352                 printk("%s: Tracepoint %d\n", ahd_name(ahd),
2353                        seqintcode - TRACEPOINT0);
2354                 break;
2355         case NO_SEQINT:
2356                 break;
2357         case SAW_HWERR:
2358                 ahd_handle_hwerrint(ahd);
2359                 break;
2360         default:
2361                 printk("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
2362                        seqintcode);
2363                 break;
2364         }
2365         /*
2366          *  The sequencer is paused immediately on
2367          *  a SEQINT, so we should restart it when
2368          *  we're done.
2369          */
2370         ahd_unpause(ahd);
2371 }
2372
2373 static void
2374 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
2375 {
2376         struct scb      *scb;
2377         u_int            status0;
2378         u_int            status3;
2379         u_int            status;
2380         u_int            lqistat1;
2381         u_int            lqostat0;
2382         u_int            scbid;
2383         u_int            busfreetime;
2384
2385         ahd_update_modes(ahd);
2386         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2387
2388         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
2389         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
2390         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
2391         lqistat1 = ahd_inb(ahd, LQISTAT1);
2392         lqostat0 = ahd_inb(ahd, LQOSTAT0);
2393         busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2394
2395         /*
2396          * Ignore external resets after a bus reset.
2397          */
2398         if (((status & SCSIRSTI) != 0) && (ahd->flags & AHD_BUS_RESET_ACTIVE)) {
2399                 ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
2400                 return;
2401         }
2402
2403         /*
2404          * Clear bus reset flag
2405          */
2406         ahd->flags &= ~AHD_BUS_RESET_ACTIVE;
2407
2408         if ((status0 & (SELDI|SELDO)) != 0) {
2409                 u_int simode0;
2410
2411                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2412                 simode0 = ahd_inb(ahd, SIMODE0);
2413                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
2414                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2415         }
2416         scbid = ahd_get_scbptr(ahd);
2417         scb = ahd_lookup_scb(ahd, scbid);
2418         if (scb != NULL
2419          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2420                 scb = NULL;
2421
2422         if ((status0 & IOERR) != 0) {
2423                 u_int now_lvd;
2424
2425                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
2426                 printk("%s: Transceiver State Has Changed to %s mode\n",
2427                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
2428                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
2429                 /*
2430                  * A change in I/O mode is equivalent to a bus reset.
2431                  */
2432                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2433                 ahd_pause(ahd);
2434                 ahd_setup_iocell_workaround(ahd);
2435                 ahd_unpause(ahd);
2436         } else if ((status0 & OVERRUN) != 0) {
2437
2438                 printk("%s: SCSI offset overrun detected.  Resetting bus.\n",
2439                        ahd_name(ahd));
2440                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2441         } else if ((status & SCSIRSTI) != 0) {
2442
2443                 printk("%s: Someone reset channel A\n", ahd_name(ahd));
2444                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
2445         } else if ((status & SCSIPERR) != 0) {
2446
2447                 /* Make sure the sequencer is in a safe location. */
2448                 ahd_clear_critical_section(ahd);
2449
2450                 ahd_handle_transmission_error(ahd);
2451         } else if (lqostat0 != 0) {
2452
2453                 printk("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
2454                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
2455                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2456                         ahd_outb(ahd, CLRLQOINT1, 0);
2457         } else if ((status & SELTO) != 0) {
2458                 /* Stop the selection */
2459                 ahd_outb(ahd, SCSISEQ0, 0);
2460
2461                 /* Make sure the sequencer is in a safe location. */
2462                 ahd_clear_critical_section(ahd);
2463
2464                 /* No more pending messages */
2465                 ahd_clear_msg_state(ahd);
2466
2467                 /* Clear interrupt state */
2468                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
2469
2470                 /*
2471                  * Although the driver does not care about the
2472                  * 'Selection in Progress' status bit, the busy
2473                  * LED does.  SELINGO is only cleared by a successful
2474                  * selection, so we must manually clear it to insure
2475                  * the LED turns off just incase no future successful
2476                  * selections occur (e.g. no devices on the bus).
2477                  */
2478                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
2479
2480                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
2481                 scb = ahd_lookup_scb(ahd, scbid);
2482                 if (scb == NULL) {
2483                         printk("%s: ahd_intr - referenced scb not "
2484                                "valid during SELTO scb(0x%x)\n",
2485                                ahd_name(ahd), scbid);
2486                         ahd_dump_card_state(ahd);
2487                 } else {
2488                         struct ahd_devinfo devinfo;
2489 #ifdef AHD_DEBUG
2490                         if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
2491                                 ahd_print_path(ahd, scb);
2492                                 printk("Saw Selection Timeout for SCB 0x%x\n",
2493                                        scbid);
2494                         }
2495 #endif
2496                         ahd_scb_devinfo(ahd, &devinfo, scb);
2497                         ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
2498                         ahd_freeze_devq(ahd, scb);
2499
2500                         /*
2501                          * Cancel any pending transactions on the device
2502                          * now that it seems to be missing.  This will
2503                          * also revert us to async/narrow transfers until
2504                          * we can renegotiate with the device.
2505                          */
2506                         ahd_handle_devreset(ahd, &devinfo,
2507                                             CAM_LUN_WILDCARD,
2508                                             CAM_SEL_TIMEOUT,
2509                                             "Selection Timeout",
2510                                             /*verbose_level*/1);
2511                 }
2512                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2513                 ahd_iocell_first_selection(ahd);
2514                 ahd_unpause(ahd);
2515         } else if ((status0 & (SELDI|SELDO)) != 0) {
2516
2517                 ahd_iocell_first_selection(ahd);
2518                 ahd_unpause(ahd);
2519         } else if (status3 != 0) {
2520                 printk("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
2521                        ahd_name(ahd), status3);
2522                 ahd_outb(ahd, CLRSINT3, status3);
2523         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
2524
2525                 /* Make sure the sequencer is in a safe location. */
2526                 ahd_clear_critical_section(ahd);
2527
2528                 ahd_handle_lqiphase_error(ahd, lqistat1);
2529         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2530                 /*
2531                  * This status can be delayed during some
2532                  * streaming operations.  The SCSIPHASE
2533                  * handler has already dealt with this case
2534                  * so just clear the error.
2535                  */
2536                 ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
2537         } else if ((status & BUSFREE) != 0
2538                 || (lqistat1 & LQOBUSFREE) != 0) {
2539                 u_int lqostat1;
2540                 int   restart;
2541                 int   clear_fifo;
2542                 int   packetized;
2543                 u_int mode;
2544
2545                 /*
2546                  * Clear our selection hardware as soon as possible.
2547                  * We may have an entry in the waiting Q for this target,
2548                  * that is affected by this busfree and we don't want to
2549                  * go about selecting the target while we handle the event.
2550                  */
2551                 ahd_outb(ahd, SCSISEQ0, 0);
2552
2553                 /* Make sure the sequencer is in a safe location. */
2554                 ahd_clear_critical_section(ahd);
2555
2556                 /*
2557                  * Determine what we were up to at the time of
2558                  * the busfree.
2559                  */
2560                 mode = AHD_MODE_SCSI;
2561                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2562                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
2563                 switch (busfreetime) {
2564                 case BUSFREE_DFF0:
2565                 case BUSFREE_DFF1:
2566                 {
2567                         mode = busfreetime == BUSFREE_DFF0
2568                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
2569                         ahd_set_modes(ahd, mode, mode);
2570                         scbid = ahd_get_scbptr(ahd);
2571                         scb = ahd_lookup_scb(ahd, scbid);
2572                         if (scb == NULL) {
2573                                 printk("%s: Invalid SCB %d in DFF%d "
2574                                        "during unexpected busfree\n",
2575                                        ahd_name(ahd), scbid, mode);
2576                                 packetized = 0;
2577                         } else
2578                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
2579                         clear_fifo = 1;
2580                         break;
2581                 }
2582                 case BUSFREE_LQO:
2583                         clear_fifo = 0;
2584                         packetized = 1;
2585                         break;
2586                 default:
2587                         clear_fifo = 0;
2588                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
2589                         if (!packetized
2590                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
2591                          && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
2592                          && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
2593                           || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
2594                                 /*
2595                                  * Assume packetized if we are not
2596                                  * on the bus in a non-packetized
2597                                  * capacity and any pending selection
2598                                  * was a packetized selection.
2599                                  */
2600                                 packetized = 1;
2601                         break;
2602                 }
2603
2604 #ifdef AHD_DEBUG
2605                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
2606                         printk("Saw Busfree.  Busfreetime = 0x%x.\n",
2607                                busfreetime);
2608 #endif
2609                 /*
2610                  * Busfrees that occur in non-packetized phases are
2611                  * handled by the nonpkt_busfree handler.
2612                  */
2613                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
2614                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
2615                 } else {
2616                         packetized = 0;
2617                         restart = ahd_handle_nonpkt_busfree(ahd);
2618                 }
2619                 /*
2620                  * Clear the busfree interrupt status.  The setting of
2621                  * the interrupt is a pulse, so in a perfect world, we
2622                  * would not need to muck with the ENBUSFREE logic.  This
2623                  * would ensure that if the bus moves on to another
2624                  * connection, busfree protection is still in force.  If
2625                  * BUSFREEREV is broken, however, we must manually clear
2626                  * the ENBUSFREE if the busfree occurred during a non-pack
2627                  * connection so that we don't get false positives during
2628                  * future, packetized, connections.
2629                  */
2630                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2631                 if (packetized == 0
2632                  && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
2633                         ahd_outb(ahd, SIMODE1,
2634                                  ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
2635
2636                 if (clear_fifo)
2637                         ahd_clear_fifo(ahd, mode);
2638
2639                 ahd_clear_msg_state(ahd);
2640                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2641                 if (restart) {
2642                         ahd_restart(ahd);
2643                 } else {
2644                         ahd_unpause(ahd);
2645                 }
2646         } else {
2647                 printk("%s: Missing case in ahd_handle_scsiint. status = %x\n",
2648                        ahd_name(ahd), status);
2649                 ahd_dump_card_state(ahd);
2650                 ahd_clear_intstat(ahd);
2651                 ahd_unpause(ahd);
2652         }
2653 }
2654
2655 static void
2656 ahd_handle_transmission_error(struct ahd_softc *ahd)
2657 {
2658         struct  scb *scb;
2659         u_int   scbid;
2660         u_int   lqistat1;
2661         u_int   msg_out;
2662         u_int   curphase;
2663         u_int   lastphase;
2664         u_int   perrdiag;
2665         u_int   cur_col;
2666         int     silent;
2667
2668         scb = NULL;
2669         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2670         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
2671         ahd_inb(ahd, LQISTAT2);
2672         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
2673          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
2674                 u_int lqistate;
2675
2676                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2677                 lqistate = ahd_inb(ahd, LQISTATE);
2678                 if ((lqistate >= 0x1E && lqistate <= 0x24)
2679                  || (lqistate == 0x29)) {
2680 #ifdef AHD_DEBUG
2681                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2682                                 printk("%s: NLQCRC found via LQISTATE\n",
2683                                        ahd_name(ahd));
2684                         }
2685 #endif
2686                         lqistat1 |= LQICRCI_NLQ;
2687                 }
2688                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2689         }
2690
2691         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2692         lastphase = ahd_inb(ahd, LASTPHASE);
2693         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2694         perrdiag = ahd_inb(ahd, PERRDIAG);
2695         msg_out = MSG_INITIATOR_DET_ERR;
2696         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
2697
2698         /*
2699          * Try to find the SCB associated with this error.
2700          */
2701         silent = FALSE;
2702         if (lqistat1 == 0
2703          || (lqistat1 & LQICRCI_NLQ) != 0) {
2704                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
2705                         ahd_set_active_fifo(ahd);
2706                 scbid = ahd_get_scbptr(ahd);
2707                 scb = ahd_lookup_scb(ahd, scbid);
2708                 if (scb != NULL && SCB_IS_SILENT(scb))
2709                         silent = TRUE;
2710         }
2711
2712         cur_col = 0;
2713         if (silent == FALSE) {
2714                 printk("%s: Transmission error detected\n", ahd_name(ahd));
2715                 ahd_lqistat1_print(lqistat1, &cur_col, 50);
2716                 ahd_lastphase_print(lastphase, &cur_col, 50);
2717                 ahd_scsisigi_print(curphase, &cur_col, 50);
2718                 ahd_perrdiag_print(perrdiag, &cur_col, 50);
2719                 printk("\n");
2720                 ahd_dump_card_state(ahd);
2721         }
2722
2723         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
2724                 if (silent == FALSE) {
2725                         printk("%s: Gross protocol error during incoming "
2726                                "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
2727                                ahd_name(ahd), lqistat1);
2728                 }
2729                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2730                 return;
2731         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
2732                 /*
2733                  * A CRC error has been detected on an incoming LQ.
2734                  * The bus is currently hung on the last ACK.
2735                  * Hit LQIRETRY to release the last ack, and
2736                  * wait for the sequencer to determine that ATNO
2737                  * is asserted while in message out to take us
2738                  * to our host message loop.  No NONPACKREQ or
2739                  * LQIPHASE type errors will occur in this
2740                  * scenario.  After this first LQIRETRY, the LQI
2741                  * manager will be in ISELO where it will
2742                  * happily sit until another packet phase begins.
2743                  * Unexpected bus free detection is enabled
2744                  * through any phases that occur after we release
2745                  * this last ack until the LQI manager sees a
2746                  * packet phase.  This implies we may have to
2747                  * ignore a perfectly valid "unexected busfree"
2748                  * after our "initiator detected error" message is
2749                  * sent.  A busfree is the expected response after
2750                  * we tell the target that it's L_Q was corrupted.
2751                  * (SPI4R09 10.7.3.3.3)
2752                  */
2753                 ahd_outb(ahd, LQCTL2, LQIRETRY);
2754                 printk("LQIRetry for LQICRCI_LQ to release ACK\n");
2755         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2756                 /*
2757                  * We detected a CRC error in a NON-LQ packet.
2758                  * The hardware has varying behavior in this situation
2759                  * depending on whether this packet was part of a
2760                  * stream or not.
2761                  *
2762                  * PKT by PKT mode:
2763                  * The hardware has already acked the complete packet.
2764                  * If the target honors our outstanding ATN condition,
2765                  * we should be (or soon will be) in MSGOUT phase.
2766                  * This will trigger the LQIPHASE_LQ status bit as the
2767                  * hardware was expecting another LQ.  Unexpected
2768                  * busfree detection is enabled.  Once LQIPHASE_LQ is
2769                  * true (first entry into host message loop is much
2770                  * the same), we must clear LQIPHASE_LQ and hit
2771                  * LQIRETRY so the hardware is ready to handle
2772                  * a future LQ.  NONPACKREQ will not be asserted again
2773                  * once we hit LQIRETRY until another packet is
2774                  * processed.  The target may either go busfree
2775                  * or start another packet in response to our message.
2776                  *
2777                  * Read Streaming P0 asserted:
2778                  * If we raise ATN and the target completes the entire
2779                  * stream (P0 asserted during the last packet), the
2780                  * hardware will ack all data and return to the ISTART
2781                  * state.  When the target reponds to our ATN condition,
2782                  * LQIPHASE_LQ will be asserted.  We should respond to
2783                  * this with an LQIRETRY to prepare for any future
2784                  * packets.  NONPACKREQ will not be asserted again
2785                  * once we hit LQIRETRY until another packet is
2786                  * processed.  The target may either go busfree or
2787                  * start another packet in response to our message.
2788                  * Busfree detection is enabled.
2789                  *
2790                  * Read Streaming P0 not asserted:
2791                  * If we raise ATN and the target transitions to
2792                  * MSGOUT in or after a packet where P0 is not
2793                  * asserted, the hardware will assert LQIPHASE_NLQ.
2794                  * We should respond to the LQIPHASE_NLQ with an
2795                  * LQIRETRY.  Should the target stay in a non-pkt
2796                  * phase after we send our message, the hardware
2797                  * will assert LQIPHASE_LQ.  Recovery is then just as
2798                  * listed above for the read streaming with P0 asserted.
2799                  * Busfree detection is enabled.
2800                  */
2801                 if (silent == FALSE)
2802                         printk("LQICRC_NLQ\n");
2803                 if (scb == NULL) {
2804                         printk("%s: No SCB valid for LQICRC_NLQ.  "
2805                                "Resetting bus\n", ahd_name(ahd));
2806                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2807                         return;
2808                 }
2809         } else if ((lqistat1 & LQIBADLQI) != 0) {
2810                 printk("Need to handle BADLQI!\n");
2811                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2812                 return;
2813         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
2814                 if ((curphase & ~P_DATAIN_DT) != 0) {
2815                         /* Ack the byte.  So we can continue. */
2816                         if (silent == FALSE)
2817                                 printk("Acking %s to clear perror\n",
2818                                     ahd_lookup_phase_entry(curphase)->phasemsg);
2819                         ahd_inb(ahd, SCSIDAT);
2820                 }
2821
2822                 if (curphase == P_MESGIN)
2823                         msg_out = MSG_PARITY_ERROR;
2824         }
2825
2826         /*
2827          * We've set the hardware to assert ATN if we
2828          * get a parity error on "in" phases, so all we
2829          * need to do is stuff the message buffer with
2830          * the appropriate message.  "In" phases have set
2831          * mesg_out to something other than MSG_NOP.
2832          */
2833         ahd->send_msg_perror = msg_out;
2834         if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
2835                 scb->flags |= SCB_TRANSMISSION_ERROR;
2836         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2837         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2838         ahd_unpause(ahd);
2839 }
2840
2841 static void
2842 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2843 {
2844         /*
2845          * Clear the sources of the interrupts.
2846          */
2847         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2848         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2849
2850         /*
2851          * If the "illegal" phase changes were in response
2852          * to our ATN to flag a CRC error, AND we ended up
2853          * on packet boundaries, clear the error, restart the
2854          * LQI manager as appropriate, and go on our merry
2855          * way toward sending the message.  Otherwise, reset
2856          * the bus to clear the error.
2857          */
2858         ahd_set_active_fifo(ahd);
2859         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2860          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2861                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
2862                         printk("LQIRETRY for LQIPHASE_LQ\n");
2863                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2864                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2865                         printk("LQIRETRY for LQIPHASE_NLQ\n");
2866                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2867                 } else
2868                         panic("ahd_handle_lqiphase_error: No phase errors\n");
2869                 ahd_dump_card_state(ahd);
2870                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2871                 ahd_unpause(ahd);
2872         } else {
2873                 printk("Resetting Channel for LQI Phase error\n");
2874                 ahd_dump_card_state(ahd);
2875                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2876         }
2877 }
2878
2879 /*
2880  * Packetized unexpected or expected busfree.
2881  * Entered in mode based on busfreetime.
2882  */
2883 static int
2884 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2885 {
2886         u_int lqostat1;
2887
2888         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2889                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2890         lqostat1 = ahd_inb(ahd, LQOSTAT1);
2891         if ((lqostat1 & LQOBUSFREE) != 0) {
2892                 struct scb *scb;
2893                 u_int scbid;
2894                 u_int saved_scbptr;
2895                 u_int waiting_h;
2896                 u_int waiting_t;
2897                 u_int next;
2898
2899                 /*
2900                  * The LQO manager detected an unexpected busfree
2901                  * either:
2902                  *
2903                  * 1) During an outgoing LQ.
2904                  * 2) After an outgoing LQ but before the first
2905                  *    REQ of the command packet.
2906                  * 3) During an outgoing command packet.
2907                  *
2908                  * In all cases, CURRSCB is pointing to the
2909                  * SCB that encountered the failure.  Clean
2910                  * up the queue, clear SELDO and LQOBUSFREE,
2911                  * and allow the sequencer to restart the select
2912                  * out at its lesure.
2913                  */
2914                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2915                 scbid = ahd_inw(ahd, CURRSCB);
2916                 scb = ahd_lookup_scb(ahd, scbid);
2917                 if (scb == NULL)
2918                        panic("SCB not valid during LQOBUSFREE");
2919                 /*
2920                  * Clear the status.
2921                  */
2922                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2923                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2924                         ahd_outb(ahd, CLRLQOINT1, 0);
2925                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2926                 ahd_flush_device_writes(ahd);
2927                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
2928
2929                 /*
2930                  * Return the LQO manager to its idle loop.  It will
2931                  * not do this automatically if the busfree occurs
2932                  * after the first REQ of either the LQ or command
2933                  * packet or between the LQ and command packet.
2934                  */
2935                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2936
2937                 /*
2938                  * Update the waiting for selection queue so
2939                  * we restart on the correct SCB.
2940                  */
2941                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2942                 saved_scbptr = ahd_get_scbptr(ahd);
2943                 if (waiting_h != scbid) {
2944
2945                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2946                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2947                         if (waiting_t == waiting_h) {
2948                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2949                                 next = SCB_LIST_NULL;
2950                         } else {
2951                                 ahd_set_scbptr(ahd, waiting_h);
2952                                 next = ahd_inw_scbram(ahd, SCB_NEXT2);
2953                         }
2954                         ahd_set_scbptr(ahd, scbid);
2955                         ahd_outw(ahd, SCB_NEXT2, next);
2956                 }
2957                 ahd_set_scbptr(ahd, saved_scbptr);
2958                 if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2959                         if (SCB_IS_SILENT(scb) == FALSE) {
2960                                 ahd_print_path(ahd, scb);
2961                                 printk("Probable outgoing LQ CRC error.  "
2962                                        "Retrying command\n");
2963                         }
2964                         scb->crc_retry_count++;
2965                 } else {
2966                         ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
2967                         ahd_freeze_scb(scb);
2968                         ahd_freeze_devq(ahd, scb);
2969                 }
2970                 /* Return unpausing the sequencer. */
2971                 return (0);
2972         } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2973                 /*
2974                  * Ignore what are really parity errors that
2975                  * occur on the last REQ of a free running
2976                  * clock prior to going busfree.  Some drives
2977                  * do not properly active negate just before
2978                  * going busfree resulting in a parity glitch.
2979                  */
2980                 ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2981 #ifdef AHD_DEBUG
2982                 if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2983                         printk("%s: Parity on last REQ detected "
2984                                "during busfree phase.\n",
2985                                ahd_name(ahd));
2986 #endif
2987                 /* Return unpausing the sequencer. */
2988                 return (0);
2989         }
2990         if (ahd->src_mode != AHD_MODE_SCSI) {
2991                 u_int   scbid;
2992                 struct  scb *scb;
2993
2994                 scbid = ahd_get_scbptr(ahd);
2995                 scb = ahd_lookup_scb(ahd, scbid);
2996                 ahd_print_path(ahd, scb);
2997                 printk("Unexpected PKT busfree condition\n");
2998                 ahd_dump_card_state(ahd);
2999                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
3000                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
3001                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
3002
3003                 /* Return restarting the sequencer. */
3004                 return (1);
3005         }
3006         printk("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
3007         ahd_dump_card_state(ahd);
3008         /* Restart the sequencer. */
3009         return (1);
3010 }
3011
3012 /*
3013  * Non-packetized unexpected or expected busfree.
3014  */
3015 static int
3016 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
3017 {
3018         struct  ahd_devinfo devinfo;
3019         struct  scb *scb;
3020         u_int   lastphase;
3021         u_int   saved_scsiid;
3022         u_int   saved_lun;
3023         u_int   target;
3024         u_int   initiator_role_id;
3025         u_int   scbid;
3026         u_int   ppr_busfree;
3027         int     printerror;
3028
3029         /*
3030          * Look at what phase we were last in.  If its message out,
3031          * chances are pretty good that the busfree was in response
3032          * to one of our abort requests.
3033          */
3034         lastphase = ahd_inb(ahd, LASTPHASE);
3035         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3036         saved_lun = ahd_inb(ahd, SAVED_LUN);
3037         target = SCSIID_TARGET(ahd, saved_scsiid);
3038         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
3039         ahd_compile_devinfo(&devinfo, initiator_role_id,
3040                             target, saved_lun, 'A', ROLE_INITIATOR);
3041         printerror = 1;
3042
3043         scbid = ahd_get_scbptr(ahd);
3044         scb = ahd_lookup_scb(ahd, scbid);
3045         if (scb != NULL
3046          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
3047                 scb = NULL;
3048
3049         ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
3050         if (lastphase == P_MESGOUT) {
3051                 u_int tag;
3052
3053                 tag = SCB_LIST_NULL;
3054                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
3055                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
3056                         int found;
3057                         int sent_msg;
3058
3059                         if (scb == NULL) {
3060                                 ahd_print_devinfo(ahd, &devinfo);
3061                                 printk("Abort for unidentified "
3062                                        "connection completed.\n");
3063                                 /* restart the sequencer. */
3064                                 return (1);
3065                         }
3066                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
3067                         ahd_print_path(ahd, scb);
3068                         printk("SCB %d - Abort%s Completed.\n",
3069                                SCB_GET_TAG(scb),
3070                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
3071
3072                         if (sent_msg == MSG_ABORT_TAG)
3073                                 tag = SCB_GET_TAG(scb);
3074
3075                         if ((scb->flags & SCB_EXTERNAL_RESET) != 0) {
3076                                 /*
3077                                  * This abort is in response to an
3078                                  * unexpected switch to command phase
3079                                  * for a packetized connection.  Since
3080                                  * the identify message was never sent,
3081                                  * "saved lun" is 0.  We really want to
3082                                  * abort only the SCB that encountered
3083                                  * this error, which could have a different
3084                                  * lun.  The SCB will be retried so the OS
3085                                  * will see the UA after renegotiating to
3086                                  * packetized.
3087                                  */
3088                                 tag = SCB_GET_TAG(scb);
3089                                 saved_lun = scb->hscb->lun;
3090                         }
3091                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
3092                                                tag, ROLE_INITIATOR,
3093                                                CAM_REQ_ABORTED);
3094                         printk("found == 0x%x\n", found);
3095                         printerror = 0;
3096                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
3097                                         MSG_BUS_DEV_RESET, TRUE)) {
3098                         ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
3099                                             CAM_BDR_SENT, "Bus Device Reset",
3100                                             /*verbose_level*/0);
3101                         printerror = 0;
3102                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
3103                         && ppr_busfree == 0) {
3104                         struct ahd_initiator_tinfo *tinfo;
3105                         struct ahd_tmode_tstate *tstate;
3106
3107                         /*
3108                          * PPR Rejected.
3109                          *
3110                          * If the previous negotiation was packetized,
3111                          * this could be because the device has been
3112                          * reset without our knowledge.  Force our
3113                          * current negotiation to async and retry the
3114                          * negotiation.  Otherwise retry the command
3115                          * with non-ppr negotiation.
3116                          */
3117 #ifdef AHD_DEBUG
3118                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3119                                 printk("PPR negotiation rejected busfree.\n");
3120 #endif
3121                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3122                                                     devinfo.our_scsiid,
3123                                                     devinfo.target, &tstate);
3124                         if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
3125                                 ahd_set_width(ahd, &devinfo,
3126                                               MSG_EXT_WDTR_BUS_8_BIT,
3127                                               AHD_TRANS_CUR,
3128                                               /*paused*/TRUE);
3129                                 ahd_set_syncrate(ahd, &devinfo,
3130                                                 /*period*/0, /*offset*/0,
3131                                                 /*ppr_options*/0,
3132                                                 AHD_TRANS_CUR,
3133                                                 /*paused*/TRUE);
3134                                 /*
3135                                  * The expect PPR busfree handler below
3136                                  * will effect the retry and necessary
3137                                  * abort.
3138                                  */
3139                         } else {
3140                                 tinfo->curr.transport_version = 2;
3141                                 tinfo->goal.transport_version = 2;
3142                                 tinfo->goal.ppr_options = 0;
3143                                 if (scb != NULL) {
3144                                         /*
3145                                          * Remove any SCBs in the waiting
3146                                          * for selection queue that may
3147                                          * also be for this target so that
3148                                          * command ordering is preserved.
3149                                          */
3150                                         ahd_freeze_devq(ahd, scb);
3151                                         ahd_qinfifo_requeue_tail(ahd, scb);
3152                                 }
3153                                 printerror = 0;
3154                         }
3155                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
3156                         && ppr_busfree == 0) {
3157                         /*
3158                          * Negotiation Rejected.  Go-narrow and
3159                          * retry command.
3160                          */
3161 #ifdef AHD_DEBUG
3162                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3163                                 printk("WDTR negotiation rejected busfree.\n");
3164 #endif
3165                         ahd_set_width(ahd, &devinfo,
3166                                       MSG_EXT_WDTR_BUS_8_BIT,
3167                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
3168                                       /*paused*/TRUE);
3169                         if (scb != NULL) {
3170                                 /*
3171                                  * Remove any SCBs in the waiting for
3172                                  * selection queue that may also be for
3173                                  * this target so that command ordering
3174                                  * is preserved.
3175                                  */
3176                                 ahd_freeze_devq(ahd, scb);
3177                                 ahd_qinfifo_requeue_tail(ahd, scb);
3178                         }
3179                         printerror = 0;
3180                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
3181                         && ppr_busfree == 0) {
3182                         /*
3183                          * Negotiation Rejected.  Go-async and
3184                          * retry command.
3185                          */
3186 #ifdef AHD_DEBUG
3187                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3188                                 printk("SDTR negotiation rejected busfree.\n");
3189 #endif
3190                         ahd_set_syncrate(ahd, &devinfo,
3191                                         /*period*/0, /*offset*/0,
3192                                         /*ppr_options*/0,
3193                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
3194                                         /*paused*/TRUE);
3195                         if (scb != NULL) {
3196                                 /*
3197                                  * Remove any SCBs in the waiting for
3198                                  * selection queue that may also be for
3199                                  * this target so that command ordering
3200                                  * is preserved.
3201                                  */
3202                                 ahd_freeze_devq(ahd, scb);
3203                                 ahd_qinfifo_requeue_tail(ahd, scb);
3204                         }
3205                         printerror = 0;
3206                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
3207                         && ahd_sent_msg(ahd, AHDMSG_1B,
3208                                          MSG_INITIATOR_DET_ERR, TRUE)) {
3209
3210 #ifdef AHD_DEBUG
3211                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3212                                 printk("Expected IDE Busfree\n");
3213 #endif
3214                         printerror = 0;
3215                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
3216                         && ahd_sent_msg(ahd, AHDMSG_1B,
3217                                         MSG_MESSAGE_REJECT, TRUE)) {
3218
3219 #ifdef AHD_DEBUG
3220                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3221                                 printk("Expected QAS Reject Busfree\n");
3222 #endif
3223                         printerror = 0;
3224                 }
3225         }
3226
3227         /*
3228          * The busfree required flag is honored at the end of
3229          * the message phases.  We check it last in case we
3230          * had to send some other message that caused a busfree.
3231          */
3232         if (scb != NULL && printerror != 0
3233          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
3234          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
3235
3236                 ahd_freeze_devq(ahd, scb);
3237                 ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
3238                 ahd_freeze_scb(scb);
3239                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
3240                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
3241                                        SCB_GET_CHANNEL(ahd, scb),
3242                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
3243                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
3244                 } else {
3245 #ifdef AHD_DEBUG
3246                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3247                                 printk("PPR Negotiation Busfree.\n");
3248 #endif
3249                         ahd_done(ahd, scb);
3250                 }
3251                 printerror = 0;
3252         }
3253         if (printerror != 0) {
3254                 int aborted;
3255
3256                 aborted = 0;
3257                 if (scb != NULL) {
3258                         u_int tag;
3259
3260                         if ((scb->hscb->control & TAG_ENB) != 0)
3261                                 tag = SCB_GET_TAG(scb);
3262                         else
3263                                 tag = SCB_LIST_NULL;
3264                         ahd_print_path(ahd, scb);
3265                         aborted = ahd_abort_scbs(ahd, target, 'A',
3266                                        SCB_GET_LUN(scb), tag,
3267                                        ROLE_INITIATOR,
3268                                        CAM_UNEXP_BUSFREE);
3269                 } else {
3270                         /*
3271                          * We had not fully identified this connection,
3272                          * so we cannot abort anything.
3273                          */
3274                         printk("%s: ", ahd_name(ahd));
3275                 }
3276                 printk("Unexpected busfree %s, %d SCBs aborted, "
3277                        "PRGMCNT == 0x%x\n",
3278                        ahd_lookup_phase_entry(lastphase)->phasemsg,
3279                        aborted,
3280                        ahd_inw(ahd, PRGMCNT));
3281                 ahd_dump_card_state(ahd);
3282                 if (lastphase != P_BUSFREE)
3283                         ahd_force_renegotiation(ahd, &devinfo);
3284         }
3285         /* Always restart the sequencer. */
3286         return (1);
3287 }
3288
3289 static void
3290 ahd_handle_proto_violation(struct ahd_softc *ahd)
3291 {
3292         struct  ahd_devinfo devinfo;
3293         struct  scb *scb;
3294         u_int   scbid;
3295         u_int   seq_flags;
3296         u_int   curphase;
3297         u_int   lastphase;
3298         int     found;
3299
3300         ahd_fetch_devinfo(ahd, &devinfo);
3301         scbid = ahd_get_scbptr(ahd);
3302         scb = ahd_lookup_scb(ahd, scbid);
3303         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
3304         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
3305         lastphase = ahd_inb(ahd, LASTPHASE);
3306         if ((seq_flags & NOT_IDENTIFIED) != 0) {
3307
3308                 /*
3309                  * The reconnecting target either did not send an
3310                  * identify message, or did, but we didn't find an SCB
3311                  * to match.
3312                  */
3313                 ahd_print_devinfo(ahd, &devinfo);
3314                 printk("Target did not send an IDENTIFY message. "
3315                        "LASTPHASE = 0x%x.\n", lastphase);
3316                 scb = NULL;
3317         } else if (scb == NULL) {
3318                 /*
3319                  * We don't seem to have an SCB active for this
3320                  * transaction.  Print an error and reset the bus.
3321                  */
3322                 ahd_print_devinfo(ahd, &devinfo);
3323                 printk("No SCB found during protocol violation\n");
3324                 goto proto_violation_reset;
3325         } else {
3326                 ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3327                 if ((seq_flags & NO_CDB_SENT) != 0) {
3328                         ahd_print_path(ahd, scb);
3329                         printk("No or incomplete CDB sent to device.\n");
3330                 } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
3331                           & STATUS_RCVD) == 0) {
3332                         /*
3333                          * The target never bothered to provide status to
3334                          * us prior to completing the command.  Since we don't
3335                          * know the disposition of this command, we must attempt
3336                          * to abort it.  Assert ATN and prepare to send an abort
3337                          * message.
3338                          */
3339                         ahd_print_path(ahd, scb);
3340                         printk("Completed command without status.\n");
3341                 } else {
3342                         ahd_print_path(ahd, scb);
3343                         printk("Unknown protocol violation.\n");
3344                         ahd_dump_card_state(ahd);
3345                 }
3346         }
3347         if ((lastphase & ~P_DATAIN_DT) == 0
3348          || lastphase == P_COMMAND) {
3349 proto_violation_reset:
3350                 /*
3351                  * Target either went directly to data
3352                  * phase or didn't respond to our ATN.
3353                  * The only safe thing to do is to blow
3354                  * it away with a bus reset.
3355                  */
3356                 found = ahd_reset_channel(ahd, 'A', TRUE);
3357                 printk("%s: Issued Channel %c Bus Reset. "
3358                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
3359         } else {
3360                 /*
3361                  * Leave the selection hardware off in case
3362                  * this abort attempt will affect yet to
3363                  * be sent commands.
3364                  */
3365                 ahd_outb(ahd, SCSISEQ0,
3366                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3367                 ahd_assert_atn(ahd);
3368                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
3369                 if (scb == NULL) {
3370                         ahd_print_devinfo(ahd, &devinfo);
3371                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
3372                         ahd->msgout_len = 1;
3373                         ahd->msgout_index = 0;
3374                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3375                 } else {
3376                         ahd_print_path(ahd, scb);
3377                         scb->flags |= SCB_ABORT;
3378                 }
3379                 printk("Protocol violation %s.  Attempting to abort.\n",
3380                        ahd_lookup_phase_entry(curphase)->phasemsg);
3381         }
3382 }
3383
3384 /*
3385  * Force renegotiation to occur the next time we initiate
3386  * a command to the current device.
3387  */
3388 static void
3389 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3390 {
3391         struct  ahd_initiator_tinfo *targ_info;
3392         struct  ahd_tmode_tstate *tstate;
3393
3394 #ifdef AHD_DEBUG
3395         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3396                 ahd_print_devinfo(ahd, devinfo);
3397                 printk("Forcing renegotiation\n");
3398         }
3399 #endif
3400         targ_info = ahd_fetch_transinfo(ahd,
3401                                         devinfo->channel,
3402                                         devinfo->our_scsiid,
3403                                         devinfo->target,
3404                                         &tstate);
3405         ahd_update_neg_request(ahd, devinfo, tstate,
3406                                targ_info, AHD_NEG_IF_NON_ASYNC);
3407 }
3408
3409 #define AHD_MAX_STEPS 2000
3410 static void
3411 ahd_clear_critical_section(struct ahd_softc *ahd)
3412 {
3413         ahd_mode_state  saved_modes;
3414         int             stepping;
3415         int             steps;
3416         int             first_instr;
3417         u_int           simode0;
3418         u_int           simode1;
3419         u_int           simode3;
3420         u_int           lqimode0;
3421         u_int           lqimode1;
3422         u_int           lqomode0;
3423         u_int           lqomode1;
3424
3425         if (ahd->num_critical_sections == 0)
3426                 return;
3427
3428         stepping = FALSE;
3429         steps = 0;
3430         first_instr = 0;
3431         simode0 = 0;
3432         simode1 = 0;
3433         simode3 = 0;
3434         lqimode0 = 0;
3435         lqimode1 = 0;
3436         lqomode0 = 0;
3437         lqomode1 = 0;
3438         saved_modes = ahd_save_modes(ahd);
3439         for (;;) {
3440                 struct  cs *cs;
3441                 u_int   seqaddr;
3442                 u_int   i;
3443
3444                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3445                 seqaddr = ahd_inw(ahd, CURADDR);
3446
3447                 cs = ahd->critical_sections;
3448                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
3449
3450                         if (cs->begin < seqaddr && cs->end >= seqaddr)
3451                                 break;
3452                 }
3453
3454                 if (i == ahd->num_critical_sections)
3455                         break;
3456
3457                 if (steps > AHD_MAX_STEPS) {
3458                         printk("%s: Infinite loop in critical section\n"
3459                                "%s: First Instruction 0x%x now 0x%x\n",
3460                                ahd_name(ahd), ahd_name(ahd), first_instr,
3461                                seqaddr);
3462                         ahd_dump_card_state(ahd);
3463                         panic("critical section loop");
3464                 }
3465
3466                 steps++;
3467 #ifdef AHD_DEBUG
3468                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
3469                         printk("%s: Single stepping at 0x%x\n", ahd_name(ahd),
3470                                seqaddr);
3471 #endif
3472                 if (stepping == FALSE) {
3473
3474                         first_instr = seqaddr;
3475                         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3476                         simode0 = ahd_inb(ahd, SIMODE0);
3477                         simode3 = ahd_inb(ahd, SIMODE3);
3478                         lqimode0 = ahd_inb(ahd, LQIMODE0);
3479                         lqimode1 = ahd_inb(ahd, LQIMODE1);
3480                         lqomode0 = ahd_inb(ahd, LQOMODE0);
3481                         lqomode1 = ahd_inb(ahd, LQOMODE1);
3482                         ahd_outb(ahd, SIMODE0, 0);
3483                         ahd_outb(ahd, SIMODE3, 0);
3484                         ahd_outb(ahd, LQIMODE0, 0);
3485                         ahd_outb(ahd, LQIMODE1, 0);
3486                         ahd_outb(ahd, LQOMODE0, 0);
3487                         ahd_outb(ahd, LQOMODE1, 0);
3488                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3489                         simode1 = ahd_inb(ahd, SIMODE1);
3490                         /*
3491                          * We don't clear ENBUSFREE.  Unfortunately
3492                          * we cannot re-enable busfree detection within
3493                          * the current connection, so we must leave it
3494                          * on while single stepping.
3495                          */
3496                         ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
3497                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
3498                         stepping = TRUE;
3499                 }
3500                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
3501                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
3502                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
3503                 ahd_outb(ahd, HCNTRL, ahd->unpause);
3504                 while (!ahd_is_paused(ahd))
3505                         ahd_delay(200);
3506                 ahd_update_modes(ahd);
3507         }
3508         if (stepping) {
3509                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3510                 ahd_outb(ahd, SIMODE0, simode0);
3511                 ahd_outb(ahd, SIMODE3, simode3);
3512                 ahd_outb(ahd, LQIMODE0, lqimode0);
3513                 ahd_outb(ahd, LQIMODE1, lqimode1);
3514                 ahd_outb(ahd, LQOMODE0, lqomode0);
3515                 ahd_outb(ahd, LQOMODE1, lqomode1);
3516                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3517                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
3518                 ahd_outb(ahd, SIMODE1, simode1);
3519                 /*
3520                  * SCSIINT seems to glitch occasionally when
3521                  * the interrupt masks are restored.  Clear SCSIINT
3522                  * one more time so that only persistent errors
3523                  * are seen as a real interrupt.
3524                  */
3525                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
3526         }
3527         ahd_restore_modes(ahd, saved_modes);
3528 }
3529
3530 /*
3531  * Clear any pending interrupt status.
3532  */
3533 static void
3534 ahd_clear_intstat(struct ahd_softc *ahd)
3535 {
3536         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
3537                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
3538         /* Clear any interrupt conditions this may have caused */
3539         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
3540                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
3541         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
3542                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
3543                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
3544         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
3545                                  |CLRLQOATNPKT|CLRLQOTCRC);
3546         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
3547                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
3548         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
3549                 ahd_outb(ahd, CLRLQOINT0, 0);
3550                 ahd_outb(ahd, CLRLQOINT1, 0);
3551         }
3552         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
3553         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
3554                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
3555         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
3556                                 |CLRIOERR|CLROVERRUN);
3557         ahd_outb(ahd, CLRINT, CLRSCSIINT);
3558 }
3559
3560 /**************************** Debugging Routines ******************************/
3561 #ifdef AHD_DEBUG
3562 uint32_t ahd_debug = AHD_DEBUG_OPTS;
3563 #endif
3564
3565 #if 0
3566 void
3567 ahd_print_scb(struct scb *scb)
3568 {
3569         struct hardware_scb *hscb;
3570         int i;
3571
3572         hscb = scb->hscb;
3573         printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
3574                (void *)scb,
3575                hscb->control,
3576                hscb->scsiid,
3577                hscb->lun,
3578                hscb->cdb_len);
3579         printk("Shared Data: ");
3580         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
3581                 printk("%#02x", hscb->shared_data.idata.cdb[i]);
3582         printk("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
3583                (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
3584                (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
3585                ahd_le32toh(hscb->datacnt),
3586                ahd_le32toh(hscb->sgptr),
3587                SCB_GET_TAG(scb));
3588         ahd_dump_sglist(scb);
3589 }
3590 #endif  /*  0  */
3591
3592 /************************* Transfer Negotiation *******************************/
3593 /*
3594  * Allocate per target mode instance (ID we respond to as a target)
3595  * transfer negotiation data structures.
3596  */
3597 static struct ahd_tmode_tstate *
3598 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
3599 {
3600         struct ahd_tmode_tstate *master_tstate;
3601         struct ahd_tmode_tstate *tstate;
3602         int i;
3603
3604         master_tstate = ahd->enabled_targets[ahd->our_id];
3605         if (ahd->enabled_targets[scsi_id] != NULL
3606          && ahd->enabled_targets[scsi_id] != master_tstate)
3607                 panic("%s: ahd_alloc_tstate - Target already allocated",
3608                       ahd_name(ahd));
3609         tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
3610         if (tstate == NULL)
3611                 return (NULL);
3612
3613         /*
3614          * If we have allocated a master tstate, copy user settings from
3615          * the master tstate (taken from SRAM or the EEPROM) for this
3616          * channel, but reset our current and goal settings to async/narrow
3617          * until an initiator talks to us.
3618          */
3619         if (master_tstate != NULL) {
3620                 memcpy(tstate, master_tstate, sizeof(*tstate));
3621                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
3622                 for (i = 0; i < 16; i++) {
3623                         memset(&tstate->transinfo[i].curr, 0,
3624                               sizeof(tstate->transinfo[i].curr));
3625                         memset(&tstate->transinfo[i].goal, 0,
3626                               sizeof(tstate->transinfo[i].goal));
3627                 }
3628         } else
3629                 memset(tstate, 0, sizeof(*tstate));
3630         ahd->enabled_targets[scsi_id] = tstate;
3631         return (tstate);
3632 }
3633
3634 #ifdef AHD_TARGET_MODE
3635 /*
3636  * Free per target mode instance (ID we respond to as a target)
3637  * transfer negotiation data structures.
3638  */
3639 static void
3640 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
3641 {
3642         struct ahd_tmode_tstate *tstate;
3643
3644         /*
3645          * Don't clean up our "master" tstate.
3646          * It has our default user settings.
3647          */
3648         if (scsi_id == ahd->our_id
3649          && force == FALSE)
3650                 return;
3651
3652         tstate = ahd->enabled_targets[scsi_id];
3653         kfree(tstate);
3654         ahd->enabled_targets[scsi_id] = NULL;
3655 }
3656 #endif
3657
3658 /*
3659  * Called when we have an active connection to a target on the bus,
3660  * this function finds the nearest period to the input period limited
3661  * by the capabilities of the bus connectivity of and sync settings for
3662  * the target.
3663  */
3664 static void
3665 ahd_devlimited_syncrate(struct ahd_softc *ahd,
3666                         struct ahd_initiator_tinfo *tinfo,
3667                         u_int *period, u_int *ppr_options, role_t role)
3668 {
3669         struct  ahd_transinfo *transinfo;
3670         u_int   maxsync;
3671
3672         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
3673          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
3674                 maxsync = AHD_SYNCRATE_PACED;
3675         } else {
3676                 maxsync = AHD_SYNCRATE_ULTRA;
3677                 /* Can't do DT related options on an SE bus */
3678                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3679         }
3680         /*
3681          * Never allow a value higher than our current goal
3682          * period otherwise we may allow a target initiated
3683          * negotiation to go above the limit as set by the
3684          * user.  In the case of an initiator initiated
3685          * sync negotiation, we limit based on the user
3686          * setting.  This allows the system to still accept
3687          * incoming negotiations even if target initiated
3688          * negotiation is not performed.
3689          */
3690         if (role == ROLE_TARGET)
3691                 transinfo = &tinfo->user;
3692         else
3693                 transinfo = &tinfo->goal;
3694         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
3695         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
3696                 maxsync = max(maxsync, (u_int)AHD_SYNCRATE_ULTRA2);
3697                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3698         }
3699         if (transinfo->period == 0) {
3700                 *period = 0;
3701                 *ppr_options = 0;
3702         } else {
3703                 *period = max(*period, (u_int)transinfo->period);
3704                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
3705         }
3706 }
3707
3708 /*
3709  * Look up the valid period to SCSIRATE conversion in our table.
3710  * Return the period and offset that should be sent to the target
3711  * if this was the beginning of an SDTR.
3712  */
3713 void
3714 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
3715                   u_int *ppr_options, u_int maxsync)
3716 {
3717         if (*period < maxsync)
3718                 *period = maxsync;
3719
3720         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
3721          && *period > AHD_SYNCRATE_MIN_DT)
3722                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3723
3724         if (*period > AHD_SYNCRATE_MIN)
3725                 *period = 0;
3726
3727         /* Honor PPR option conformance rules. */
3728         if (*period > AHD_SYNCRATE_PACED)
3729                 *ppr_options &= ~MSG_EXT_PPR_RTI;
3730
3731         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3732                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
3733
3734         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
3735                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3736
3737         /* Skip all PACED only entries if IU is not available */
3738         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
3739          && *period < AHD_SYNCRATE_DT)
3740                 *period = AHD_SYNCRATE_DT;
3741
3742         /* Skip all DT only entries if DT is not available */
3743         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3744          && *period < AHD_SYNCRATE_ULTRA2)
3745                 *period = AHD_SYNCRATE_ULTRA2;
3746 }
3747
3748 /*
3749  * Truncate the given synchronous offset to a value the
3750  * current adapter type and syncrate are capable of.
3751  */
3752 static void
3753 ahd_validate_offset(struct ahd_softc *ahd,
3754                     struct ahd_initiator_tinfo *tinfo,
3755                     u_int period, u_int *offset, int wide,
3756                     role_t role)
3757 {
3758         u_int maxoffset;
3759
3760         /* Limit offset to what we can do */
3761         if (period == 0)
3762                 maxoffset = 0;
3763         else if (period <= AHD_SYNCRATE_PACED) {
3764                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
3765                         maxoffset = MAX_OFFSET_PACED_BUG;
3766                 else
3767                         maxoffset = MAX_OFFSET_PACED;
3768         } else
3769                 maxoffset = MAX_OFFSET_NON_PACED;
3770         *offset = min(*offset, maxoffset);
3771         if (tinfo != NULL) {
3772                 if (role == ROLE_TARGET)
3773                         *offset = min(*offset, (u_int)tinfo->user.offset);
3774                 else
3775                         *offset = min(*offset, (u_int)tinfo->goal.offset);
3776         }
3777 }
3778
3779 /*
3780  * Truncate the given transfer width parameter to a value the
3781  * current adapter type is capable of.
3782  */
3783 static void
3784 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3785                    u_int *bus_width, role_t role)
3786 {
3787         switch (*bus_width) {
3788         default:
3789                 if (ahd->features & AHD_WIDE) {
3790                         /* Respond Wide */
3791                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3792                         break;
3793                 }
3794                 fallthrough;
3795         case MSG_EXT_WDTR_BUS_8_BIT:
3796                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3797                 break;
3798         }
3799         if (tinfo != NULL) {
3800                 if (role == ROLE_TARGET)
3801                         *bus_width = min((u_int)tinfo->user.width, *bus_width);
3802                 else
3803                         *bus_width = min((u_int)tinfo->goal.width, *bus_width);
3804         }
3805 }
3806
3807 /*
3808  * Update the bitmask of targets for which the controller should
3809  * negotiate with at the next convenient opportunity.  This currently
3810  * means the next time we send the initial identify messages for
3811  * a new transaction.
3812  */
3813 int
3814 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3815                        struct ahd_tmode_tstate *tstate,
3816                        struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3817 {
3818         u_int auto_negotiate_orig;
3819
3820         auto_negotiate_orig = tstate->auto_negotiate;
3821         if (neg_type == AHD_NEG_ALWAYS) {
3822                 /*
3823                  * Force our "current" settings to be
3824                  * unknown so that unless a bus reset
3825                  * occurs the need to renegotiate is
3826                  * recorded persistently.
3827                  */
3828                 if ((ahd->features & AHD_WIDE) != 0)
3829                         tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3830                 tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3831                 tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3832         }
3833         if (tinfo->curr.period != tinfo->goal.period
3834          || tinfo->curr.width != tinfo->goal.width
3835          || tinfo->curr.offset != tinfo->goal.offset
3836          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3837          || (neg_type == AHD_NEG_IF_NON_ASYNC
3838           && (tinfo->goal.offset != 0
3839            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3840            || tinfo->goal.ppr_options != 0)))
3841                 tstate->auto_negotiate |= devinfo->target_mask;
3842         else
3843                 tstate->auto_negotiate &= ~devinfo->target_mask;
3844
3845         return (auto_negotiate_orig != tstate->auto_negotiate);
3846 }
3847
3848 /*
3849  * Update the user/goal/curr tables of synchronous negotiation
3850  * parameters as well as, in the case of a current or active update,
3851  * any data structures on the host controller.  In the case of an
3852  * active update, the specified target is currently talking to us on
3853  * the bus, so the transfer parameter update must take effect
3854  * immediately.
3855  */
3856 void
3857 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3858                  u_int period, u_int offset, u_int ppr_options,
3859                  u_int type, int paused)
3860 {
3861         struct  ahd_initiator_tinfo *tinfo;
3862         struct  ahd_tmode_tstate *tstate;
3863         u_int   old_period;
3864         u_int   old_offset;
3865         u_int   old_ppr;
3866         int     active;
3867         int     update_needed;
3868
3869         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3870         update_needed = 0;
3871
3872         if (period == 0 || offset == 0) {
3873                 period = 0;
3874                 offset = 0;
3875         }
3876
3877         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3878                                     devinfo->target, &tstate);
3879
3880         if ((type & AHD_TRANS_USER) != 0) {
3881                 tinfo->user.period = period;
3882                 tinfo->user.offset = offset;
3883                 tinfo->user.ppr_options = ppr_options;
3884         }
3885
3886         if ((type & AHD_TRANS_GOAL) != 0) {
3887                 tinfo->goal.period = period;
3888                 tinfo->goal.offset = offset;
3889                 tinfo->goal.ppr_options = ppr_options;
3890         }
3891
3892         old_period = tinfo->curr.period;
3893         old_offset = tinfo->curr.offset;
3894         old_ppr    = tinfo->curr.ppr_options;
3895
3896         if ((type & AHD_TRANS_CUR) != 0
3897          && (old_period != period
3898           || old_offset != offset
3899           || old_ppr != ppr_options)) {
3900
3901                 update_needed++;
3902
3903                 tinfo->curr.period = period;
3904                 tinfo->curr.offset = offset;
3905                 tinfo->curr.ppr_options = ppr_options;
3906
3907                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3908                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
3909                 if (bootverbose) {
3910                         if (offset != 0) {
3911                                 int options;
3912
3913                                 printk("%s: target %d synchronous with "
3914                                        "period = 0x%x, offset = 0x%x",
3915                                        ahd_name(ahd), devinfo->target,
3916                                        period, offset);
3917                                 options = 0;
3918                                 if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3919                                         printk("(RDSTRM");
3920                                         options++;
3921                                 }
3922                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3923                                         printk("%s", options ? "|DT" : "(DT");
3924                                         options++;
3925                                 }
3926                                 if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3927                                         printk("%s", options ? "|IU" : "(IU");
3928                                         options++;
3929                                 }
3930                                 if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3931                                         printk("%s", options ? "|RTI" : "(RTI");
3932                                         options++;
3933                                 }
3934                                 if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3935                                         printk("%s", options ? "|QAS" : "(QAS");
3936                                         options++;
3937                                 }
3938                                 if (options != 0)
3939                                         printk(")\n");
3940                                 else
3941                                         printk("\n");
3942                         } else {
3943                                 printk("%s: target %d using "
3944                                        "asynchronous transfers%s\n",
3945                                        ahd_name(ahd), devinfo->target,
3946                                        (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3947                                      ?  "(QAS)" : "");
3948                         }
3949                 }
3950         }
3951         /*
3952          * Always refresh the neg-table to handle the case of the
3953          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3954          * We will always renegotiate in that case if this is a
3955          * packetized request.  Also manage the busfree expected flag
3956          * from this common routine so that we catch changes due to
3957          * WDTR or SDTR messages.
3958          */
3959         if ((type & AHD_TRANS_CUR) != 0) {
3960                 if (!paused)
3961                         ahd_pause(ahd);
3962                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3963                 if (!paused)
3964                         ahd_unpause(ahd);
3965                 if (ahd->msg_type != MSG_TYPE_NONE) {
3966                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3967                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3968 #ifdef AHD_DEBUG
3969                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3970                                         ahd_print_devinfo(ahd, devinfo);
3971                                         printk("Expecting IU Change busfree\n");
3972                                 }
3973 #endif
3974                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3975                                                |  MSG_FLAG_IU_REQ_CHANGED;
3976                         }
3977                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3978 #ifdef AHD_DEBUG
3979                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3980                                         printk("PPR with IU_REQ outstanding\n");
3981 #endif
3982                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3983                         }
3984                 }
3985         }
3986
3987         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3988                                                 tinfo, AHD_NEG_TO_GOAL);
3989
3990         if (update_needed && active)
3991                 ahd_update_pending_scbs(ahd);
3992 }
3993
3994 /*
3995  * Update the user/goal/curr tables of wide negotiation
3996  * parameters as well as, in the case of a current or active update,
3997  * any data structures on the host controller.  In the case of an
3998  * active update, the specified target is currently talking to us on
3999  * the bus, so the transfer parameter update must take effect
4000  * immediately.
4001  */
4002 void
4003 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4004               u_int width, u_int type, int paused)
4005 {
4006         struct  ahd_initiator_tinfo *tinfo;
4007         struct  ahd_tmode_tstate *tstate;
4008         u_int   oldwidth;
4009         int     active;
4010         int     update_needed;
4011
4012         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
4013         update_needed = 0;
4014         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4015                                     devinfo->target, &tstate);
4016
4017         if ((type & AHD_TRANS_USER) != 0)
4018                 tinfo->user.width = width;
4019
4020         if ((type & AHD_TRANS_GOAL) != 0)
4021                 tinfo->goal.width = width;
4022
4023         oldwidth = tinfo->curr.width;
4024         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
4025
4026                 update_needed++;
4027
4028                 tinfo->curr.width = width;
4029                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
4030                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
4031                 if (bootverbose) {
4032                         printk("%s: target %d using %dbit transfers\n",
4033                                ahd_name(ahd), devinfo->target,
4034                                8 * (0x01 << width));
4035                 }
4036         }
4037
4038         if ((type & AHD_TRANS_CUR) != 0) {
4039                 if (!paused)
4040                         ahd_pause(ahd);
4041                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
4042                 if (!paused)
4043                         ahd_unpause(ahd);
4044         }
4045
4046         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
4047                                                 tinfo, AHD_NEG_TO_GOAL);
4048         if (update_needed && active)
4049                 ahd_update_pending_scbs(ahd);
4050
4051 }
4052
4053 /*
4054  * Update the current state of tagged queuing for a given target.
4055  */
4056 static void
4057 ahd_set_tags(struct ahd_softc *ahd, struct scsi_cmnd *cmd,
4058              struct ahd_devinfo *devinfo, ahd_queue_alg alg)
4059 {
4060         struct scsi_device *sdev = cmd->device;
4061
4062         ahd_platform_set_tags(ahd, sdev, devinfo, alg);
4063         ahd_send_async(ahd, devinfo->channel, devinfo->target,
4064                        devinfo->lun, AC_TRANSFER_NEG);
4065 }
4066
4067 static void
4068 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4069                      struct ahd_transinfo *tinfo)
4070 {
4071         ahd_mode_state  saved_modes;
4072         u_int           period;
4073         u_int           ppr_opts;
4074         u_int           con_opts;
4075         u_int           offset;
4076         u_int           saved_negoaddr;
4077         uint8_t         iocell_opts[sizeof(ahd->iocell_opts)];
4078
4079         saved_modes = ahd_save_modes(ahd);
4080         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4081
4082         saved_negoaddr = ahd_inb(ahd, NEGOADDR);
4083         ahd_outb(ahd, NEGOADDR, devinfo->target);
4084         period = tinfo->period;
4085         offset = tinfo->offset;
4086         memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
4087         ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
4088                                         |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
4089         con_opts = 0;
4090         if (period == 0)
4091                 period = AHD_SYNCRATE_ASYNC;
4092         if (period == AHD_SYNCRATE_160) {
4093
4094                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4095                         /*
4096                          * When the SPI4 spec was finalized, PACE transfers
4097                          * was not made a configurable option in the PPR
4098                          * message.  Instead it is assumed to be enabled for
4099                          * any syncrate faster than 80MHz.  Nevertheless,
4100                          * Harpoon2A4 allows this to be configurable.
4101                          *
4102                          * Harpoon2A4 also assumes at most 2 data bytes per
4103                          * negotiated REQ/ACK offset.  Paced transfers take
4104                          * 4, so we must adjust our offset.
4105                          */
4106                         ppr_opts |= PPROPT_PACE;
4107                         offset *= 2;
4108
4109                         /*
4110                          * Harpoon2A assumed that there would be a
4111                          * fallback rate between 160MHz and 80MHz,
4112                          * so 7 is used as the period factor rather
4113                          * than 8 for 160MHz.
4114                          */
4115                         period = AHD_SYNCRATE_REVA_160;
4116                 }
4117                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
4118                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4119                             ~AHD_PRECOMP_MASK;
4120         } else {
4121                 /*
4122                  * Precomp should be disabled for non-paced transfers.
4123                  */
4124                 iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
4125
4126                 if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
4127                  && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
4128                  && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
4129                         /*
4130                          * Slow down our CRC interval to be
4131                          * compatible with non-packetized
4132                          * U160 devices that can't handle a
4133                          * CRC at full speed.
4134                          */
4135                         con_opts |= ENSLOWCRC;
4136                 }
4137
4138                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4139                         /*
4140                          * On H2A4, revert to a slower slewrate
4141                          * on non-paced transfers.
4142                          */
4143                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4144                             ~AHD_SLEWRATE_MASK;
4145                 }
4146         }
4147
4148         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
4149         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
4150         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
4151         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
4152
4153         ahd_outb(ahd, NEGPERIOD, period);
4154         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
4155         ahd_outb(ahd, NEGOFFSET, offset);
4156
4157         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
4158                 con_opts |= WIDEXFER;
4159
4160         /*
4161          * Slow down our CRC interval to be
4162          * compatible with packetized U320 devices
4163          * that can't handle a CRC at full speed
4164          */
4165         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
4166                 con_opts |= ENSLOWCRC;
4167         }
4168
4169         /*
4170          * During packetized transfers, the target will
4171          * give us the opportunity to send command packets
4172          * without us asserting attention.
4173          */
4174         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
4175                 con_opts |= ENAUTOATNO;
4176         ahd_outb(ahd, NEGCONOPTS, con_opts);
4177         ahd_outb(ahd, NEGOADDR, saved_negoaddr);
4178         ahd_restore_modes(ahd, saved_modes);
4179 }
4180
4181 /*
4182  * When the transfer settings for a connection change, setup for
4183  * negotiation in pending SCBs to effect the change as quickly as
4184  * possible.  We also cancel any negotiations that are scheduled
4185  * for inflight SCBs that have not been started yet.
4186  */
4187 static void
4188 ahd_update_pending_scbs(struct ahd_softc *ahd)
4189 {
4190         struct          scb *pending_scb;
4191         int             pending_scb_count;
4192         int             paused;
4193         u_int           saved_scbptr;
4194         ahd_mode_state  saved_modes;
4195
4196         /*
4197          * Traverse the pending SCB list and ensure that all of the
4198          * SCBs there have the proper settings.  We can only safely
4199          * clear the negotiation required flag (setting requires the
4200          * execution queue to be modified) and this is only possible
4201          * if we are not already attempting to select out for this
4202          * SCB.  For this reason, all callers only call this routine
4203          * if we are changing the negotiation settings for the currently
4204          * active transaction on the bus.
4205          */
4206         pending_scb_count = 0;
4207         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4208                 struct ahd_devinfo devinfo;
4209                 struct ahd_tmode_tstate *tstate;
4210
4211                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
4212                 ahd_fetch_transinfo(ahd, devinfo.channel, devinfo.our_scsiid,
4213                                     devinfo.target, &tstate);
4214                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
4215                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
4216                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
4217                         pending_scb->hscb->control &= ~MK_MESSAGE;
4218                 }
4219                 ahd_sync_scb(ahd, pending_scb,
4220                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
4221                 pending_scb_count++;
4222         }
4223
4224         if (pending_scb_count == 0)
4225                 return;
4226
4227         if (ahd_is_paused(ahd)) {
4228                 paused = 1;
4229         } else {
4230                 paused = 0;
4231                 ahd_pause(ahd);
4232         }
4233
4234         /*
4235          * Force the sequencer to reinitialize the selection for
4236          * the command at the head of the execution queue if it
4237          * has already been setup.  The negotiation changes may
4238          * effect whether we select-out with ATN.  It is only
4239          * safe to clear ENSELO when the bus is not free and no
4240          * selection is in progres or completed.
4241          */
4242         saved_modes = ahd_save_modes(ahd);
4243         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4244         if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
4245          && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
4246                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
4247         saved_scbptr = ahd_get_scbptr(ahd);
4248         /* Ensure that the hscbs down on the card match the new information */
4249         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4250                 u_int   scb_tag;
4251                 u_int   control;
4252
4253                 scb_tag = SCB_GET_TAG(pending_scb);
4254                 ahd_set_scbptr(ahd, scb_tag);
4255                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
4256                 control &= ~MK_MESSAGE;
4257                 control |= pending_scb->hscb->control & MK_MESSAGE;
4258                 ahd_outb(ahd, SCB_CONTROL, control);
4259         }
4260         ahd_set_scbptr(ahd, saved_scbptr);
4261         ahd_restore_modes(ahd, saved_modes);
4262
4263         if (paused == 0)
4264                 ahd_unpause(ahd);
4265 }
4266
4267 /**************************** Pathing Information *****************************/
4268 static void
4269 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4270 {
4271         ahd_mode_state  saved_modes;
4272         u_int           saved_scsiid;
4273         role_t          role;
4274         int             our_id;
4275
4276         saved_modes = ahd_save_modes(ahd);
4277         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4278
4279         if (ahd_inb(ahd, SSTAT0) & TARGET)
4280                 role = ROLE_TARGET;
4281         else
4282                 role = ROLE_INITIATOR;
4283
4284         if (role == ROLE_TARGET
4285          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
4286                 /* We were selected, so pull our id from TARGIDIN */
4287                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
4288         } else if (role == ROLE_TARGET)
4289                 our_id = ahd_inb(ahd, TOWNID);
4290         else
4291                 our_id = ahd_inb(ahd, IOWNID);
4292
4293         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
4294         ahd_compile_devinfo(devinfo,
4295                             our_id,
4296                             SCSIID_TARGET(ahd, saved_scsiid),
4297                             ahd_inb(ahd, SAVED_LUN),
4298                             SCSIID_CHANNEL(ahd, saved_scsiid),
4299                             role);
4300         ahd_restore_modes(ahd, saved_modes);
4301 }
4302
4303 void
4304 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4305 {
4306         printk("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
4307                devinfo->target, devinfo->lun);
4308 }
4309
4310 static const struct ahd_phase_table_entry*
4311 ahd_lookup_phase_entry(int phase)
4312 {
4313         const struct ahd_phase_table_entry *entry;
4314         const struct ahd_phase_table_entry *last_entry;
4315
4316         /*
4317          * num_phases doesn't include the default entry which
4318          * will be returned if the phase doesn't match.
4319          */
4320         last_entry = &ahd_phase_table[num_phases];
4321         for (entry = ahd_phase_table; entry < last_entry; entry++) {
4322                 if (phase == entry->phase)
4323                         break;
4324         }
4325         return (entry);
4326 }
4327
4328 void
4329 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
4330                     u_int lun, char channel, role_t role)
4331 {
4332         devinfo->our_scsiid = our_id;
4333         devinfo->target = target;
4334         devinfo->lun = lun;
4335         devinfo->target_offset = target;
4336         devinfo->channel = channel;
4337         devinfo->role = role;
4338         if (channel == 'B')
4339                 devinfo->target_offset += 8;
4340         devinfo->target_mask = (0x01 << devinfo->target_offset);
4341 }
4342
4343 static void
4344 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4345                 struct scb *scb)
4346 {
4347         role_t  role;
4348         int     our_id;
4349
4350         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
4351         role = ROLE_INITIATOR;
4352         if ((scb->hscb->control & TARGET_SCB) != 0)
4353                 role = ROLE_TARGET;
4354         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
4355                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
4356 }
4357
4358
4359 /************************ Message Phase Processing ****************************/
4360 /*
4361  * When an initiator transaction with the MK_MESSAGE flag either reconnects
4362  * or enters the initial message out phase, we are interrupted.  Fill our
4363  * outgoing message buffer with the appropriate message and beging handing
4364  * the message phase(s) manually.
4365  */
4366 static void
4367 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4368                            struct scb *scb)
4369 {
4370         /*
4371          * To facilitate adding multiple messages together,
4372          * each routine should increment the index and len
4373          * variables instead of setting them explicitly.
4374          */
4375         ahd->msgout_index = 0;
4376         ahd->msgout_len = 0;
4377
4378         if (ahd_currently_packetized(ahd))
4379                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
4380
4381         if (ahd->send_msg_perror
4382          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
4383                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
4384                 ahd->msgout_len++;
4385                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4386 #ifdef AHD_DEBUG
4387                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4388                         printk("Setting up for Parity Error delivery\n");
4389 #endif
4390                 return;
4391         } else if (scb == NULL) {
4392                 printk("%s: WARNING. No pending message for "
4393                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
4394                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
4395                 ahd->msgout_len++;
4396                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4397                 return;
4398         }
4399
4400         if ((scb->flags & SCB_DEVICE_RESET) == 0
4401          && (scb->flags & SCB_PACKETIZED) == 0
4402          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
4403                 u_int identify_msg;
4404
4405                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
4406                 if ((scb->hscb->control & DISCENB) != 0)
4407                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
4408                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
4409                 ahd->msgout_len++;
4410
4411                 if ((scb->hscb->control & TAG_ENB) != 0) {
4412                         ahd->msgout_buf[ahd->msgout_index++] =
4413                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
4414                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
4415                         ahd->msgout_len += 2;
4416                 }
4417         }
4418
4419         if (scb->flags & SCB_DEVICE_RESET) {
4420                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
4421                 ahd->msgout_len++;
4422                 ahd_print_path(ahd, scb);
4423                 printk("Bus Device Reset Message Sent\n");
4424                 /*
4425                  * Clear our selection hardware in advance of
4426                  * the busfree.  We may have an entry in the waiting
4427                  * Q for this target, and we don't want to go about
4428                  * selecting while we handle the busfree and blow it
4429                  * away.
4430                  */
4431                 ahd_outb(ahd, SCSISEQ0, 0);
4432         } else if ((scb->flags & SCB_ABORT) != 0) {
4433
4434                 if ((scb->hscb->control & TAG_ENB) != 0) {
4435                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
4436                 } else {
4437                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
4438                 }
4439                 ahd->msgout_len++;
4440                 ahd_print_path(ahd, scb);
4441                 printk("Abort%s Message Sent\n",
4442                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
4443                 /*
4444                  * Clear our selection hardware in advance of
4445                  * the busfree.  We may have an entry in the waiting
4446                  * Q for this target, and we don't want to go about
4447                  * selecting while we handle the busfree and blow it
4448                  * away.
4449                  */
4450                 ahd_outb(ahd, SCSISEQ0, 0);
4451         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
4452                 ahd_build_transfer_msg(ahd, devinfo);
4453                 /*
4454                  * Clear our selection hardware in advance of potential
4455                  * PPR IU status change busfree.  We may have an entry in
4456                  * the waiting Q for this target, and we don't want to go
4457                  * about selecting while we handle the busfree and blow
4458                  * it away.
4459                  */
4460                 ahd_outb(ahd, SCSISEQ0, 0);
4461         } else {
4462                 printk("ahd_intr: AWAITING_MSG for an SCB that "
4463                        "does not have a waiting message\n");
4464                 printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
4465                        devinfo->target_mask);
4466                 panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
4467                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
4468                       ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
4469                       scb->flags);
4470         }
4471
4472         /*
4473          * Clear the MK_MESSAGE flag from the SCB so we aren't
4474          * asked to send this message again.
4475          */
4476         ahd_outb(ahd, SCB_CONTROL,
4477                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
4478         scb->hscb->control &= ~MK_MESSAGE;
4479         ahd->msgout_index = 0;
4480         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4481 }
4482
4483 /*
4484  * Build an appropriate transfer negotiation message for the
4485  * currently active target.
4486  */
4487 static void
4488 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4489 {
4490         /*
4491          * We need to initiate transfer negotiations.
4492          * If our current and goal settings are identical,
4493          * we want to renegotiate due to a check condition.
4494          */
4495         struct  ahd_initiator_tinfo *tinfo;
4496         struct  ahd_tmode_tstate *tstate;
4497         int     dowide;
4498         int     dosync;
4499         int     doppr;
4500         u_int   period;
4501         u_int   ppr_options;
4502         u_int   offset;
4503
4504         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4505                                     devinfo->target, &tstate);
4506         /*
4507          * Filter our period based on the current connection.
4508          * If we can't perform DT transfers on this segment (not in LVD
4509          * mode for instance), then our decision to issue a PPR message
4510          * may change.
4511          */
4512         period = tinfo->goal.period;
4513         offset = tinfo->goal.offset;
4514         ppr_options = tinfo->goal.ppr_options;
4515         /* Target initiated PPR is not allowed in the SCSI spec */
4516         if (devinfo->role == ROLE_TARGET)
4517                 ppr_options = 0;
4518         ahd_devlimited_syncrate(ahd, tinfo, &period,
4519                                 &ppr_options, devinfo->role);
4520         dowide = tinfo->curr.width != tinfo->goal.width;
4521         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
4522         /*
4523          * Only use PPR if we have options that need it, even if the device
4524          * claims to support it.  There might be an expander in the way
4525          * that doesn't.
4526          */
4527         doppr = ppr_options != 0;
4528
4529         if (!dowide && !dosync && !doppr) {
4530                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
4531                 dosync = tinfo->goal.offset != 0;
4532         }
4533
4534         if (!dowide && !dosync && !doppr) {
4535                 /*
4536                  * Force async with a WDTR message if we have a wide bus,
4537                  * or just issue an SDTR with a 0 offset.
4538                  */
4539                 if ((ahd->features & AHD_WIDE) != 0)
4540                         dowide = 1;
4541                 else
4542                         dosync = 1;
4543
4544                 if (bootverbose) {
4545                         ahd_print_devinfo(ahd, devinfo);
4546                         printk("Ensuring async\n");
4547                 }
4548         }
4549         /* Target initiated PPR is not allowed in the SCSI spec */
4550         if (devinfo->role == ROLE_TARGET)
4551                 doppr = 0;
4552
4553         /*
4554          * Both the PPR message and SDTR message require the
4555          * goal syncrate to be limited to what the target device
4556          * is capable of handling (based on whether an LVD->SE
4557          * expander is on the bus), so combine these two cases.
4558          * Regardless, guarantee that if we are using WDTR and SDTR
4559          * messages that WDTR comes first.
4560          */
4561         if (doppr || (dosync && !dowide)) {
4562
4563                 offset = tinfo->goal.offset;
4564                 ahd_validate_offset(ahd, tinfo, period, &offset,
4565                                     doppr ? tinfo->goal.width
4566                                           : tinfo->curr.width,
4567                                     devinfo->role);
4568                 if (doppr) {
4569                         ahd_construct_ppr(ahd, devinfo, period, offset,
4570                                           tinfo->goal.width, ppr_options);
4571                 } else {
4572                         ahd_construct_sdtr(ahd, devinfo, period, offset);
4573                 }
4574         } else {
4575                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
4576         }
4577 }
4578
4579 /*
4580  * Build a synchronous negotiation message in our message
4581  * buffer based on the input parameters.
4582  */
4583 static void
4584 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4585                    u_int period, u_int offset)
4586 {
4587         if (offset == 0)
4588                 period = AHD_ASYNC_XFER_PERIOD;
4589         ahd->msgout_index += spi_populate_sync_msg(
4590                         ahd->msgout_buf + ahd->msgout_index, period, offset);
4591         ahd->msgout_len += 5;
4592         if (bootverbose) {
4593                 printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
4594                        ahd_name(ahd), devinfo->channel, devinfo->target,
4595                        devinfo->lun, period, offset);
4596         }
4597 }
4598
4599 /*
4600  * Build a wide negotiateion message in our message
4601  * buffer based on the input parameters.
4602  */
4603 static void
4604 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4605                    u_int bus_width)
4606 {
4607         ahd->msgout_index += spi_populate_width_msg(
4608                         ahd->msgout_buf + ahd->msgout_index, bus_width);
4609         ahd->msgout_len += 4;
4610         if (bootverbose) {
4611                 printk("(%s:%c:%d:%d): Sending WDTR %x\n",
4612                        ahd_name(ahd), devinfo->channel, devinfo->target,
4613                        devinfo->lun, bus_width);
4614         }
4615 }
4616
4617 /*
4618  * Build a parallel protocol request message in our message
4619  * buffer based on the input parameters.
4620  */
4621 static void
4622 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4623                   u_int period, u_int offset, u_int bus_width,
4624                   u_int ppr_options)
4625 {
4626         /*
4627          * Always request precompensation from
4628          * the other target if we are running
4629          * at paced syncrates.
4630          */
4631         if (period <= AHD_SYNCRATE_PACED)
4632                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
4633         if (offset == 0)
4634                 period = AHD_ASYNC_XFER_PERIOD;
4635         ahd->msgout_index += spi_populate_ppr_msg(
4636                         ahd->msgout_buf + ahd->msgout_index, period, offset,
4637                         bus_width, ppr_options);
4638         ahd->msgout_len += 8;
4639         if (bootverbose) {
4640                 printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
4641                        "offset %x, ppr_options %x\n", ahd_name(ahd),
4642                        devinfo->channel, devinfo->target, devinfo->lun,
4643                        bus_width, period, offset, ppr_options);
4644         }
4645 }
4646
4647 /*
4648  * Clear any active message state.
4649  */
4650 static void
4651 ahd_clear_msg_state(struct ahd_softc *ahd)
4652 {
4653         ahd_mode_state saved_modes;
4654
4655         saved_modes = ahd_save_modes(ahd);
4656         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4657         ahd->send_msg_perror = 0;
4658         ahd->msg_flags = MSG_FLAG_NONE;
4659         ahd->msgout_len = 0;
4660         ahd->msgin_index = 0;
4661         ahd->msg_type = MSG_TYPE_NONE;
4662         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
4663                 /*
4664                  * The target didn't care to respond to our
4665                  * message request, so clear ATN.
4666                  */
4667                 ahd_outb(ahd, CLRSINT1, CLRATNO);
4668         }
4669         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
4670         ahd_outb(ahd, SEQ_FLAGS2,
4671                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
4672         ahd_restore_modes(ahd, saved_modes);
4673 }
4674
4675 /*
4676  * Manual message loop handler.
4677  */
4678 static void
4679 ahd_handle_message_phase(struct ahd_softc *ahd)
4680 {
4681         struct  ahd_devinfo devinfo;
4682         u_int   bus_phase;
4683         int     end_session;
4684
4685         ahd_fetch_devinfo(ahd, &devinfo);
4686         end_session = FALSE;
4687         bus_phase = ahd_inb(ahd, LASTPHASE);
4688
4689         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
4690                 printk("LQIRETRY for LQIPHASE_OUTPKT\n");
4691                 ahd_outb(ahd, LQCTL2, LQIRETRY);
4692         }
4693 reswitch:
4694         switch (ahd->msg_type) {
4695         case MSG_TYPE_INITIATOR_MSGOUT:
4696         {
4697                 int lastbyte;
4698                 int phasemis;
4699                 int msgdone;
4700
4701                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
4702                         panic("HOST_MSG_LOOP interrupt with no active message");
4703
4704 #ifdef AHD_DEBUG
4705                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4706                         ahd_print_devinfo(ahd, &devinfo);
4707                         printk("INITIATOR_MSG_OUT");
4708                 }
4709 #endif
4710                 phasemis = bus_phase != P_MESGOUT;
4711                 if (phasemis) {
4712 #ifdef AHD_DEBUG
4713                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4714                                 printk(" PHASEMIS %s\n",
4715                                        ahd_lookup_phase_entry(bus_phase)
4716                                                              ->phasemsg);
4717                         }
4718 #endif
4719                         if (bus_phase == P_MESGIN) {
4720                                 /*
4721                                  * Change gears and see if
4722                                  * this messages is of interest to
4723                                  * us or should be passed back to
4724                                  * the sequencer.
4725                                  */
4726                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
4727                                 ahd->send_msg_perror = 0;
4728                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
4729                                 ahd->msgin_index = 0;
4730                                 goto reswitch;
4731                         }
4732                         end_session = TRUE;
4733                         break;
4734                 }
4735
4736                 if (ahd->send_msg_perror) {
4737                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4738                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4739 #ifdef AHD_DEBUG
4740                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4741                                 printk(" byte 0x%x\n", ahd->send_msg_perror);
4742 #endif
4743                         /*
4744                          * If we are notifying the target of a CRC error
4745                          * during packetized operations, the target is
4746                          * within its rights to acknowledge our message
4747                          * with a busfree.
4748                          */
4749                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
4750                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
4751                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
4752
4753                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
4754                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4755                         break;
4756                 }
4757
4758                 msgdone = ahd->msgout_index == ahd->msgout_len;
4759                 if (msgdone) {
4760                         /*
4761                          * The target has requested a retry.
4762                          * Re-assert ATN, reset our message index to
4763                          * 0, and try again.
4764                          */
4765                         ahd->msgout_index = 0;
4766                         ahd_assert_atn(ahd);
4767                 }
4768
4769                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
4770                 if (lastbyte) {
4771                         /* Last byte is signified by dropping ATN */
4772                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4773                 }
4774
4775                 /*
4776                  * Clear our interrupt status and present
4777                  * the next byte on the bus.
4778                  */
4779                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4780 #ifdef AHD_DEBUG
4781                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4782                         printk(" byte 0x%x\n",
4783                                ahd->msgout_buf[ahd->msgout_index]);
4784 #endif
4785                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
4786                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4787                 break;
4788         }
4789         case MSG_TYPE_INITIATOR_MSGIN:
4790         {
4791                 int phasemis;
4792                 int message_done;
4793
4794 #ifdef AHD_DEBUG
4795                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4796                         ahd_print_devinfo(ahd, &devinfo);
4797                         printk("INITIATOR_MSG_IN");
4798                 }
4799 #endif
4800                 phasemis = bus_phase != P_MESGIN;
4801                 if (phasemis) {
4802 #ifdef AHD_DEBUG
4803                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4804                                 printk(" PHASEMIS %s\n",
4805                                        ahd_lookup_phase_entry(bus_phase)
4806                                                              ->phasemsg);
4807                         }
4808 #endif
4809                         ahd->msgin_index = 0;
4810                         if (bus_phase == P_MESGOUT
4811                          && (ahd->send_msg_perror != 0
4812                           || (ahd->msgout_len != 0
4813                            && ahd->msgout_index == 0))) {
4814                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4815                                 goto reswitch;
4816                         }
4817                         end_session = TRUE;
4818                         break;
4819                 }
4820
4821                 /* Pull the byte in without acking it */
4822                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4823 #ifdef AHD_DEBUG
4824                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4825                         printk(" byte 0x%x\n",
4826                                ahd->msgin_buf[ahd->msgin_index]);
4827 #endif
4828
4829                 message_done = ahd_parse_msg(ahd, &devinfo);
4830
4831                 if (message_done) {
4832                         /*
4833                          * Clear our incoming message buffer in case there
4834                          * is another message following this one.
4835                          */
4836                         ahd->msgin_index = 0;
4837
4838                         /*
4839                          * If this message illicited a response,
4840                          * assert ATN so the target takes us to the
4841                          * message out phase.
4842                          */
4843                         if (ahd->msgout_len != 0) {
4844 #ifdef AHD_DEBUG
4845                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4846                                         ahd_print_devinfo(ahd, &devinfo);
4847                                         printk("Asserting ATN for response\n");
4848                                 }
4849 #endif
4850                                 ahd_assert_atn(ahd);
4851                         }
4852                 } else
4853                         ahd->msgin_index++;
4854
4855                 if (message_done == MSGLOOP_TERMINATED) {
4856                         end_session = TRUE;
4857                 } else {
4858                         /* Ack the byte */
4859                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4860                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4861                 }
4862                 break;
4863         }
4864         case MSG_TYPE_TARGET_MSGIN:
4865         {
4866                 int msgdone;
4867                 int msgout_request;
4868
4869                 /*
4870                  * By default, the message loop will continue.
4871                  */
4872                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4873
4874                 if (ahd->msgout_len == 0)
4875                         panic("Target MSGIN with no active message");
4876
4877                 /*
4878                  * If we interrupted a mesgout session, the initiator
4879                  * will not know this until our first REQ.  So, we
4880                  * only honor mesgout requests after we've sent our
4881                  * first byte.
4882                  */
4883                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4884                  && ahd->msgout_index > 0)
4885                         msgout_request = TRUE;
4886                 else
4887                         msgout_request = FALSE;
4888
4889                 if (msgout_request) {
4890
4891                         /*
4892                          * Change gears and see if
4893                          * this messages is of interest to
4894                          * us or should be passed back to
4895                          * the sequencer.
4896                          */
4897                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4898                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4899                         ahd->msgin_index = 0;
4900                         /* Dummy read to REQ for first byte */
4901                         ahd_inb(ahd, SCSIDAT);
4902                         ahd_outb(ahd, SXFRCTL0,
4903                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4904                         break;
4905                 }
4906
4907                 msgdone = ahd->msgout_index == ahd->msgout_len;
4908                 if (msgdone) {
4909                         ahd_outb(ahd, SXFRCTL0,
4910                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4911                         end_session = TRUE;
4912                         break;
4913                 }
4914
4915                 /*
4916                  * Present the next byte on the bus.
4917                  */
4918                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4919                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4920                 break;
4921         }
4922         case MSG_TYPE_TARGET_MSGOUT:
4923         {
4924                 int lastbyte;
4925                 int msgdone;
4926
4927                 /*
4928                  * By default, the message loop will continue.
4929                  */
4930                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4931
4932                 /*
4933                  * The initiator signals that this is
4934                  * the last byte by dropping ATN.
4935                  */
4936                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4937
4938                 /*
4939                  * Read the latched byte, but turn off SPIOEN first
4940                  * so that we don't inadvertently cause a REQ for the
4941                  * next byte.
4942                  */
4943                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4944                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4945                 msgdone = ahd_parse_msg(ahd, &devinfo);
4946                 if (msgdone == MSGLOOP_TERMINATED) {
4947                         /*
4948                          * The message is *really* done in that it caused
4949                          * us to go to bus free.  The sequencer has already
4950                          * been reset at this point, so pull the ejection
4951                          * handle.
4952                          */
4953                         return;
4954                 }
4955
4956                 ahd->msgin_index++;
4957
4958                 /*
4959                  * XXX Read spec about initiator dropping ATN too soon
4960                  *     and use msgdone to detect it.
4961                  */
4962                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
4963                         ahd->msgin_index = 0;
4964
4965                         /*
4966                          * If this message illicited a response, transition
4967                          * to the Message in phase and send it.
4968                          */
4969                         if (ahd->msgout_len != 0) {
4970                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
4971                                 ahd_outb(ahd, SXFRCTL0,
4972                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4973                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4974                                 ahd->msgin_index = 0;
4975                                 break;
4976                         }
4977                 }
4978
4979                 if (lastbyte)
4980                         end_session = TRUE;
4981                 else {
4982                         /* Ask for the next byte. */
4983                         ahd_outb(ahd, SXFRCTL0,
4984                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4985                 }
4986
4987                 break;
4988         }
4989         default:
4990                 panic("Unknown REQINIT message type");
4991         }
4992
4993         if (end_session) {
4994                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
4995                         printk("%s: Returning to Idle Loop\n",
4996                                ahd_name(ahd));
4997                         ahd_clear_msg_state(ahd);
4998
4999                         /*
5000                          * Perform the equivalent of a clear_target_state.
5001                          */
5002                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
5003                         ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
5004                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
5005                 } else {
5006                         ahd_clear_msg_state(ahd);
5007                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
5008                 }
5009         }
5010 }
5011
5012 /*
5013  * See if we sent a particular extended message to the target.
5014  * If "full" is true, return true only if the target saw the full
5015  * message.  If "full" is false, return true if the target saw at
5016  * least the first byte of the message.
5017  */
5018 static int
5019 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
5020 {
5021         int found;
5022         u_int index;
5023
5024         found = FALSE;
5025         index = 0;
5026
5027         while (index < ahd->msgout_len) {
5028                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
5029                         u_int end_index;
5030
5031                         end_index = index + 1 + ahd->msgout_buf[index + 1];
5032                         if (ahd->msgout_buf[index+2] == msgval
5033                          && type == AHDMSG_EXT) {
5034
5035                                 if (full) {
5036                                         if (ahd->msgout_index > end_index)
5037                                                 found = TRUE;
5038                                 } else if (ahd->msgout_index > index)
5039                                         found = TRUE;
5040                         }
5041                         index = end_index;
5042                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
5043                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
5044
5045                         /* Skip tag type and tag id or residue param*/
5046                         index += 2;
5047                 } else {
5048                         /* Single byte message */
5049                         if (type == AHDMSG_1B
5050                          && ahd->msgout_index > index
5051                          && (ahd->msgout_buf[index] == msgval
5052                           || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
5053                            && msgval == MSG_IDENTIFYFLAG)))
5054                                 found = TRUE;
5055                         index++;
5056                 }
5057
5058                 if (found)
5059                         break;
5060         }
5061         return (found);
5062 }
5063
5064 /*
5065  * Wait for a complete incoming message, parse it, and respond accordingly.
5066  */
5067 static int
5068 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5069 {
5070         struct  ahd_initiator_tinfo *tinfo;
5071         struct  ahd_tmode_tstate *tstate;
5072         int     reject;
5073         int     done;
5074         int     response;
5075
5076         done = MSGLOOP_IN_PROG;
5077         response = FALSE;
5078         reject = FALSE;
5079         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
5080                                     devinfo->target, &tstate);
5081
5082         /*
5083          * Parse as much of the message as is available,
5084          * rejecting it if we don't support it.  When
5085          * the entire message is available and has been
5086          * handled, return MSGLOOP_MSGCOMPLETE, indicating
5087          * that we have parsed an entire message.
5088          *
5089          * In the case of extended messages, we accept the length
5090          * byte outright and perform more checking once we know the
5091          * extended message type.
5092          */
5093         switch (ahd->msgin_buf[0]) {
5094         case MSG_DISCONNECT:
5095         case MSG_SAVEDATAPOINTER:
5096         case MSG_CMDCOMPLETE:
5097         case MSG_RESTOREPOINTERS:
5098         case MSG_IGN_WIDE_RESIDUE:
5099                 /*
5100                  * End our message loop as these are messages
5101                  * the sequencer handles on its own.
5102                  */
5103                 done = MSGLOOP_TERMINATED;
5104                 break;
5105         case MSG_MESSAGE_REJECT:
5106                 response = ahd_handle_msg_reject(ahd, devinfo);
5107                 fallthrough;
5108         case MSG_NOOP:
5109                 done = MSGLOOP_MSGCOMPLETE;
5110                 break;
5111         case MSG_EXTENDED:
5112         {
5113                 /* Wait for enough of the message to begin validation */
5114                 if (ahd->msgin_index < 2)
5115                         break;
5116                 switch (ahd->msgin_buf[2]) {
5117                 case MSG_EXT_SDTR:
5118                 {
5119                         u_int    period;
5120                         u_int    ppr_options;
5121                         u_int    offset;
5122                         u_int    saved_offset;
5123
5124                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
5125                                 reject = TRUE;
5126                                 break;
5127                         }
5128
5129                         /*
5130                          * Wait until we have both args before validating
5131                          * and acting on this message.
5132                          *
5133                          * Add one to MSG_EXT_SDTR_LEN to account for
5134                          * the extended message preamble.
5135                          */
5136                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
5137                                 break;
5138
5139                         period = ahd->msgin_buf[3];
5140                         ppr_options = 0;
5141                         saved_offset = offset = ahd->msgin_buf[4];
5142                         ahd_devlimited_syncrate(ahd, tinfo, &period,
5143                                                 &ppr_options, devinfo->role);
5144                         ahd_validate_offset(ahd, tinfo, period, &offset,
5145                                             tinfo->curr.width, devinfo->role);
5146                         if (bootverbose) {
5147                                 printk("(%s:%c:%d:%d): Received "
5148                                        "SDTR period %x, offset %x\n\t"
5149                                        "Filtered to period %x, offset %x\n",
5150                                        ahd_name(ahd), devinfo->channel,
5151                                        devinfo->target, devinfo->lun,
5152                                        ahd->msgin_buf[3], saved_offset,
5153                                        period, offset);
5154                         }
5155                         ahd_set_syncrate(ahd, devinfo, period,
5156                                          offset, ppr_options,
5157                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5158                                          /*paused*/TRUE);
5159
5160                         /*
5161                          * See if we initiated Sync Negotiation
5162                          * and didn't have to fall down to async
5163                          * transfers.
5164                          */
5165                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
5166                                 /* We started it */
5167                                 if (saved_offset != offset) {
5168                                         /* Went too low - force async */
5169                                         reject = TRUE;
5170                                 }
5171                         } else {
5172                                 /*
5173                                  * Send our own SDTR in reply
5174                                  */
5175                                 if (bootverbose
5176                                  && devinfo->role == ROLE_INITIATOR) {
5177                                         printk("(%s:%c:%d:%d): Target "
5178                                                "Initiated SDTR\n",
5179                                                ahd_name(ahd), devinfo->channel,
5180                                                devinfo->target, devinfo->lun);
5181                                 }
5182                                 ahd->msgout_index = 0;
5183                                 ahd->msgout_len = 0;
5184                                 ahd_construct_sdtr(ahd, devinfo,
5185                                                    period, offset);
5186                                 ahd->msgout_index = 0;
5187                                 response = TRUE;
5188                         }
5189                         done = MSGLOOP_MSGCOMPLETE;
5190                         break;
5191                 }
5192                 case MSG_EXT_WDTR:
5193                 {
5194                         u_int bus_width;
5195                         u_int saved_width;
5196                         u_int sending_reply;
5197
5198                         sending_reply = FALSE;
5199                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
5200                                 reject = TRUE;
5201                                 break;
5202                         }
5203
5204                         /*
5205                          * Wait until we have our arg before validating
5206                          * and acting on this message.
5207                          *
5208                          * Add one to MSG_EXT_WDTR_LEN to account for
5209                          * the extended message preamble.
5210                          */
5211                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
5212                                 break;
5213
5214                         bus_width = ahd->msgin_buf[3];
5215                         saved_width = bus_width;
5216                         ahd_validate_width(ahd, tinfo, &bus_width,
5217                                            devinfo->role);
5218                         if (bootverbose) {
5219                                 printk("(%s:%c:%d:%d): Received WDTR "
5220                                        "%x filtered to %x\n",
5221                                        ahd_name(ahd), devinfo->channel,
5222                                        devinfo->target, devinfo->lun,
5223                                        saved_width, bus_width);
5224                         }
5225
5226                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
5227                                 /*
5228                                  * Don't send a WDTR back to the
5229                                  * target, since we asked first.
5230                                  * If the width went higher than our
5231                                  * request, reject it.
5232                                  */
5233                                 if (saved_width > bus_width) {
5234                                         reject = TRUE;
5235                                         printk("(%s:%c:%d:%d): requested %dBit "
5236                                                "transfers.  Rejecting...\n",
5237                                                ahd_name(ahd), devinfo->channel,
5238                                                devinfo->target, devinfo->lun,
5239                                                8 * (0x01 << bus_width));
5240                                         bus_width = 0;
5241                                 }
5242                         } else {
5243                                 /*
5244                                  * Send our own WDTR in reply
5245                                  */
5246                                 if (bootverbose
5247                                  && devinfo->role == ROLE_INITIATOR) {
5248                                         printk("(%s:%c:%d:%d): Target "
5249                                                "Initiated WDTR\n",
5250                                                ahd_name(ahd), devinfo->channel,
5251                                                devinfo->target, devinfo->lun);
5252                                 }
5253                                 ahd->msgout_index = 0;
5254                                 ahd->msgout_len = 0;
5255                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
5256                                 ahd->msgout_index = 0;
5257                                 response = TRUE;
5258                                 sending_reply = TRUE;
5259                         }
5260                         /*
5261                          * After a wide message, we are async, but
5262                          * some devices don't seem to honor this portion
5263                          * of the spec.  Force a renegotiation of the
5264                          * sync component of our transfer agreement even
5265                          * if our goal is async.  By updating our width
5266                          * after forcing the negotiation, we avoid
5267                          * renegotiating for width.
5268                          */
5269                         ahd_update_neg_request(ahd, devinfo, tstate,
5270                                                tinfo, AHD_NEG_ALWAYS);
5271                         ahd_set_width(ahd, devinfo, bus_width,
5272                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5273                                       /*paused*/TRUE);
5274                         if (sending_reply == FALSE && reject == FALSE) {
5275
5276                                 /*
5277                                  * We will always have an SDTR to send.
5278                                  */
5279                                 ahd->msgout_index = 0;
5280                                 ahd->msgout_len = 0;
5281                                 ahd_build_transfer_msg(ahd, devinfo);
5282                                 ahd->msgout_index = 0;
5283                                 response = TRUE;
5284                         }
5285                         done = MSGLOOP_MSGCOMPLETE;
5286                         break;
5287                 }
5288                 case MSG_EXT_PPR:
5289                 {
5290                         u_int   period;
5291                         u_int   offset;
5292                         u_int   bus_width;
5293                         u_int   ppr_options;
5294                         u_int   saved_width;
5295                         u_int   saved_offset;
5296                         u_int   saved_ppr_options;
5297
5298                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
5299                                 reject = TRUE;
5300                                 break;
5301                         }
5302
5303                         /*
5304                          * Wait until we have all args before validating
5305                          * and acting on this message.
5306                          *
5307                          * Add one to MSG_EXT_PPR_LEN to account for
5308                          * the extended message preamble.
5309                          */
5310                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
5311                                 break;
5312
5313                         period = ahd->msgin_buf[3];
5314                         offset = ahd->msgin_buf[5];
5315                         bus_width = ahd->msgin_buf[6];
5316                         saved_width = bus_width;
5317                         ppr_options = ahd->msgin_buf[7];
5318                         /*
5319                          * According to the spec, a DT only
5320                          * period factor with no DT option
5321                          * set implies async.
5322                          */
5323                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
5324                          && period <= 9)
5325                                 offset = 0;
5326                         saved_ppr_options = ppr_options;
5327                         saved_offset = offset;
5328
5329                         /*
5330                          * Transfer options are only available if we
5331                          * are negotiating wide.
5332                          */
5333                         if (bus_width == 0)
5334                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
5335
5336                         ahd_validate_width(ahd, tinfo, &bus_width,
5337                                            devinfo->role);
5338                         ahd_devlimited_syncrate(ahd, tinfo, &period,
5339                                                 &ppr_options, devinfo->role);
5340                         ahd_validate_offset(ahd, tinfo, period, &offset,
5341                                             bus_width, devinfo->role);
5342
5343                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
5344                                 /*
5345                                  * If we are unable to do any of the
5346                                  * requested options (we went too low),
5347                                  * then we'll have to reject the message.
5348                                  */
5349                                 if (saved_width > bus_width
5350                                  || saved_offset != offset
5351                                  || saved_ppr_options != ppr_options) {
5352                                         reject = TRUE;
5353                                         period = 0;
5354                                         offset = 0;
5355                                         bus_width = 0;
5356                                         ppr_options = 0;
5357                                 }
5358                         } else {
5359                                 if (devinfo->role != ROLE_TARGET)
5360                                         printk("(%s:%c:%d:%d): Target "
5361                                                "Initiated PPR\n",
5362                                                ahd_name(ahd), devinfo->channel,
5363                                                devinfo->target, devinfo->lun);
5364                                 else
5365                                         printk("(%s:%c:%d:%d): Initiator "
5366                                                "Initiated PPR\n",
5367                                                ahd_name(ahd), devinfo->channel,
5368                                                devinfo->target, devinfo->lun);
5369                                 ahd->msgout_index = 0;
5370                                 ahd->msgout_len = 0;
5371                                 ahd_construct_ppr(ahd, devinfo, period, offset,
5372                                                   bus_width, ppr_options);
5373                                 ahd->msgout_index = 0;
5374                                 response = TRUE;
5375                         }
5376                         if (bootverbose) {
5377                                 printk("(%s:%c:%d:%d): Received PPR width %x, "
5378                                        "period %x, offset %x,options %x\n"
5379                                        "\tFiltered to width %x, period %x, "
5380                                        "offset %x, options %x\n",
5381                                        ahd_name(ahd), devinfo->channel,
5382                                        devinfo->target, devinfo->lun,
5383                                        saved_width, ahd->msgin_buf[3],
5384                                        saved_offset, saved_ppr_options,
5385                                        bus_width, period, offset, ppr_options);
5386                         }
5387                         ahd_set_width(ahd, devinfo, bus_width,
5388                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5389                                       /*paused*/TRUE);
5390                         ahd_set_syncrate(ahd, devinfo, period,
5391                                          offset, ppr_options,
5392                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5393                                          /*paused*/TRUE);
5394
5395                         done = MSGLOOP_MSGCOMPLETE;
5396                         break;
5397                 }
5398                 default:
5399                         /* Unknown extended message.  Reject it. */
5400                         reject = TRUE;
5401                         break;
5402                 }
5403                 break;
5404         }
5405 #ifdef AHD_TARGET_MODE
5406         case MSG_BUS_DEV_RESET:
5407                 ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
5408                                     CAM_BDR_SENT,
5409                                     "Bus Device Reset Received",
5410                                     /*verbose_level*/0);
5411                 ahd_restart(ahd);
5412                 done = MSGLOOP_TERMINATED;
5413                 break;
5414         case MSG_ABORT_TAG:
5415         case MSG_ABORT:
5416         case MSG_CLEAR_QUEUE:
5417         {
5418                 int tag;
5419
5420                 /* Target mode messages */
5421                 if (devinfo->role != ROLE_TARGET) {
5422                         reject = TRUE;
5423                         break;
5424                 }
5425                 tag = SCB_LIST_NULL;
5426                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
5427                         tag = ahd_inb(ahd, INITIATOR_TAG);
5428                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5429                                devinfo->lun, tag, ROLE_TARGET,
5430                                CAM_REQ_ABORTED);
5431
5432                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
5433                 if (tstate != NULL) {
5434                         struct ahd_tmode_lstate* lstate;
5435
5436                         lstate = tstate->enabled_luns[devinfo->lun];
5437                         if (lstate != NULL) {
5438                                 ahd_queue_lstate_event(ahd, lstate,
5439                                                        devinfo->our_scsiid,
5440                                                        ahd->msgin_buf[0],
5441                                                        /*arg*/tag);
5442                                 ahd_send_lstate_events(ahd, lstate);
5443                         }
5444                 }
5445                 ahd_restart(ahd);
5446                 done = MSGLOOP_TERMINATED;
5447                 break;
5448         }
5449 #endif
5450         case MSG_QAS_REQUEST:
5451 #ifdef AHD_DEBUG
5452                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
5453                         printk("%s: QAS request.  SCSISIGI == 0x%x\n",
5454                                ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
5455 #endif
5456                 ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
5457                 fallthrough;
5458         case MSG_TERM_IO_PROC:
5459         default:
5460                 reject = TRUE;
5461                 break;
5462         }
5463
5464         if (reject) {
5465                 /*
5466                  * Setup to reject the message.
5467                  */
5468                 ahd->msgout_index = 0;
5469                 ahd->msgout_len = 1;
5470                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
5471                 done = MSGLOOP_MSGCOMPLETE;
5472                 response = TRUE;
5473         }
5474
5475         if (done != MSGLOOP_IN_PROG && !response)
5476                 /* Clear the outgoing message buffer */
5477                 ahd->msgout_len = 0;
5478
5479         return (done);
5480 }
5481
5482 /*
5483  * Process a message reject message.
5484  */
5485 static int
5486 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5487 {
5488         /*
5489          * What we care about here is if we had an
5490          * outstanding SDTR or WDTR message for this
5491          * target.  If we did, this is a signal that
5492          * the target is refusing negotiation.
5493          */
5494         struct scb *scb;
5495         struct ahd_initiator_tinfo *tinfo;
5496         struct ahd_tmode_tstate *tstate;
5497         u_int scb_index;
5498         u_int last_msg;
5499         int   response = 0;
5500
5501         scb_index = ahd_get_scbptr(ahd);
5502         scb = ahd_lookup_scb(ahd, scb_index);
5503         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
5504                                     devinfo->our_scsiid,
5505                                     devinfo->target, &tstate);
5506         /* Might be necessary */
5507         last_msg = ahd_inb(ahd, LAST_MSG);
5508
5509         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
5510                 if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
5511                  && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
5512                         /*
5513                          * Target may not like our SPI-4 PPR Options.
5514                          * Attempt to negotiate 80MHz which will turn
5515                          * off these options.
5516                          */
5517                         if (bootverbose) {
5518                                 printk("(%s:%c:%d:%d): PPR Rejected. "
5519                                        "Trying simple U160 PPR\n",
5520                                        ahd_name(ahd), devinfo->channel,
5521                                        devinfo->target, devinfo->lun);
5522                         }
5523                         tinfo->goal.period = AHD_SYNCRATE_DT;
5524                         tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
5525                                                 |  MSG_EXT_PPR_QAS_REQ
5526                                                 |  MSG_EXT_PPR_DT_REQ;
5527                 } else {
5528                         /*
5529                          * Target does not support the PPR message.
5530                          * Attempt to negotiate SPI-2 style.
5531                          */
5532                         if (bootverbose) {
5533                                 printk("(%s:%c:%d:%d): PPR Rejected. "
5534                                        "Trying WDTR/SDTR\n",
5535                                        ahd_name(ahd), devinfo->channel,
5536                                        devinfo->target, devinfo->lun);
5537                         }
5538                         tinfo->goal.ppr_options = 0;
5539                         tinfo->curr.transport_version = 2;
5540                         tinfo->goal.transport_version = 2;
5541                 }
5542                 ahd->msgout_index = 0;
5543                 ahd->msgout_len = 0;
5544                 ahd_build_transfer_msg(ahd, devinfo);
5545                 ahd->msgout_index = 0;
5546                 response = 1;
5547         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
5548
5549                 /* note 8bit xfers */
5550                 printk("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
5551                        "8bit transfers\n", ahd_name(ahd),
5552                        devinfo->channel, devinfo->target, devinfo->lun);
5553                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5554                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5555                               /*paused*/TRUE);
5556                 /*
5557                  * No need to clear the sync rate.  If the target
5558                  * did not accept the command, our syncrate is
5559                  * unaffected.  If the target started the negotiation,
5560                  * but rejected our response, we already cleared the
5561                  * sync rate before sending our WDTR.
5562                  */
5563                 if (tinfo->goal.offset != tinfo->curr.offset) {
5564
5565                         /* Start the sync negotiation */
5566                         ahd->msgout_index = 0;
5567                         ahd->msgout_len = 0;
5568                         ahd_build_transfer_msg(ahd, devinfo);
5569                         ahd->msgout_index = 0;
5570                         response = 1;
5571                 }
5572         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
5573                 /* note asynch xfers and clear flag */
5574                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
5575                                  /*offset*/0, /*ppr_options*/0,
5576                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5577                                  /*paused*/TRUE);
5578                 printk("(%s:%c:%d:%d): refuses synchronous negotiation. "
5579                        "Using asynchronous transfers\n",
5580                        ahd_name(ahd), devinfo->channel,
5581                        devinfo->target, devinfo->lun);
5582         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
5583                 int tag_type;
5584                 int mask;
5585
5586                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
5587
5588                 if (tag_type == MSG_SIMPLE_TASK) {
5589                         printk("(%s:%c:%d:%d): refuses tagged commands.  "
5590                                "Performing non-tagged I/O\n", ahd_name(ahd),
5591                                devinfo->channel, devinfo->target, devinfo->lun);
5592                         ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_NONE);
5593                         mask = ~0x23;
5594                 } else {
5595                         printk("(%s:%c:%d:%d): refuses %s tagged commands.  "
5596                                "Performing simple queue tagged I/O only\n",
5597                                ahd_name(ahd), devinfo->channel, devinfo->target,
5598                                devinfo->lun, tag_type == MSG_ORDERED_TASK
5599                                ? "ordered" : "head of queue");
5600                         ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_BASIC);
5601                         mask = ~0x03;
5602                 }
5603
5604                 /*
5605                  * Resend the identify for this CCB as the target
5606                  * may believe that the selection is invalid otherwise.
5607                  */
5608                 ahd_outb(ahd, SCB_CONTROL,
5609                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
5610                 scb->hscb->control &= mask;
5611                 ahd_set_transaction_tag(scb, /*enabled*/FALSE,
5612                                         /*type*/MSG_SIMPLE_TASK);
5613                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
5614                 ahd_assert_atn(ahd);
5615                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
5616                              SCB_GET_TAG(scb));
5617
5618                 /*
5619                  * Requeue all tagged commands for this target
5620                  * currently in our possession so they can be
5621                  * converted to untagged commands.
5622                  */
5623                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
5624                                    SCB_GET_CHANNEL(ahd, scb),
5625                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
5626                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
5627                                    SEARCH_COMPLETE);
5628         } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
5629                 /*
5630                  * Most likely the device believes that we had
5631                  * previously negotiated packetized.
5632                  */
5633                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
5634                                |  MSG_FLAG_IU_REQ_CHANGED;
5635
5636                 ahd_force_renegotiation(ahd, devinfo);
5637                 ahd->msgout_index = 0;
5638                 ahd->msgout_len = 0;
5639                 ahd_build_transfer_msg(ahd, devinfo);
5640                 ahd->msgout_index = 0;
5641                 response = 1;
5642         } else {
5643                 /*
5644                  * Otherwise, we ignore it.
5645                  */
5646                 printk("%s:%c:%d: Message reject for %x -- ignored\n",
5647                        ahd_name(ahd), devinfo->channel, devinfo->target,
5648                        last_msg);
5649         }
5650         return (response);
5651 }
5652
5653 /*
5654  * Process an ingnore wide residue message.
5655  */
5656 static void
5657 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5658 {
5659         u_int scb_index;
5660         struct scb *scb;
5661
5662         scb_index = ahd_get_scbptr(ahd);
5663         scb = ahd_lookup_scb(ahd, scb_index);
5664         /*
5665          * XXX Actually check data direction in the sequencer?
5666          * Perhaps add datadir to some spare bits in the hscb?
5667          */
5668         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
5669          || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
5670                 /*
5671                  * Ignore the message if we haven't
5672                  * seen an appropriate data phase yet.
5673                  */
5674         } else {
5675                 /*
5676                  * If the residual occurred on the last
5677                  * transfer and the transfer request was
5678                  * expected to end on an odd count, do
5679                  * nothing.  Otherwise, subtract a byte
5680                  * and update the residual count accordingly.
5681                  */
5682                 uint32_t sgptr;
5683
5684                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
5685                 if ((sgptr & SG_LIST_NULL) != 0
5686                  && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5687                      & SCB_XFERLEN_ODD) != 0) {
5688                         /*
5689                          * If the residual occurred on the last
5690                          * transfer and the transfer request was
5691                          * expected to end on an odd count, do
5692                          * nothing.
5693                          */
5694                 } else {
5695                         uint32_t data_cnt;
5696                         uint64_t data_addr;
5697                         uint32_t sglen;
5698
5699                         /* Pull in the rest of the sgptr */
5700                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5701                         data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
5702                         if ((sgptr & SG_LIST_NULL) != 0) {
5703                                 /*
5704                                  * The residual data count is not updated
5705                                  * for the command run to completion case.
5706                                  * Explicitly zero the count.
5707                                  */
5708                                 data_cnt &= ~AHD_SG_LEN_MASK;
5709                         }
5710                         data_addr = ahd_inq(ahd, SHADDR);
5711                         data_cnt += 1;
5712                         data_addr -= 1;
5713                         sgptr &= SG_PTR_MASK;
5714                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5715                                 struct ahd_dma64_seg *sg;
5716
5717                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5718
5719                                 /*
5720                                  * The residual sg ptr points to the next S/G
5721                                  * to load so we must go back one.
5722                                  */
5723                                 sg--;
5724                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5725                                 if (sg != scb->sg_list
5726                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5727
5728                                         sg--;
5729                                         sglen = ahd_le32toh(sg->len);
5730                                         /*
5731                                          * Preserve High Address and SG_LIST
5732                                          * bits while setting the count to 1.
5733                                          */
5734                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5735                                         data_addr = ahd_le64toh(sg->addr)
5736                                                   + (sglen & AHD_SG_LEN_MASK)
5737                                                   - 1;
5738
5739                                         /*
5740                                          * Increment sg so it points to the
5741                                          * "next" sg.
5742                                          */
5743                                         sg++;
5744                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5745                                                                    sg);
5746                                 }
5747                         } else {
5748                                 struct ahd_dma_seg *sg;
5749
5750                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5751
5752                                 /*
5753                                  * The residual sg ptr points to the next S/G
5754                                  * to load so we must go back one.
5755                                  */
5756                                 sg--;
5757                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5758                                 if (sg != scb->sg_list
5759                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5760
5761                                         sg--;
5762                                         sglen = ahd_le32toh(sg->len);
5763                                         /*
5764                                          * Preserve High Address and SG_LIST
5765                                          * bits while setting the count to 1.
5766                                          */
5767                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5768                                         data_addr = ahd_le32toh(sg->addr)
5769                                                   + (sglen & AHD_SG_LEN_MASK)
5770                                                   - 1;
5771
5772                                         /*
5773                                          * Increment sg so it points to the
5774                                          * "next" sg.
5775                                          */
5776                                         sg++;
5777                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5778                                                                   sg);
5779                                 }
5780                         }
5781                         /*
5782                          * Toggle the "oddness" of the transfer length
5783                          * to handle this mid-transfer ignore wide
5784                          * residue.  This ensures that the oddness is
5785                          * correct for subsequent data transfers.
5786                          */
5787                         ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
5788                             ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5789                             ^ SCB_XFERLEN_ODD);
5790
5791                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
5792                         ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
5793                         /*
5794                          * The FIFO's pointers will be updated if/when the
5795                          * sequencer re-enters a data phase.
5796                          */
5797                 }
5798         }
5799 }
5800
5801
5802 /*
5803  * Reinitialize the data pointers for the active transfer
5804  * based on its current residual.
5805  */
5806 static void
5807 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
5808 {
5809         struct           scb *scb;
5810         ahd_mode_state   saved_modes;
5811         u_int            scb_index;
5812         u_int            wait;
5813         uint32_t         sgptr;
5814         uint32_t         resid;
5815         uint64_t         dataptr;
5816
5817         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5818                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5819
5820         scb_index = ahd_get_scbptr(ahd);
5821         scb = ahd_lookup_scb(ahd, scb_index);
5822
5823         /*
5824          * Release and reacquire the FIFO so we
5825          * have a clean slate.
5826          */
5827         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5828         wait = 1000;
5829         while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5830                 ahd_delay(100);
5831         if (wait == 0) {
5832                 ahd_print_path(ahd, scb);
5833                 printk("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5834                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5835         }
5836         saved_modes = ahd_save_modes(ahd);
5837         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5838         ahd_outb(ahd, DFFSTAT,
5839                  ahd_inb(ahd, DFFSTAT)
5840                 | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5841
5842         /*
5843          * Determine initial values for data_addr and data_cnt
5844          * for resuming the data phase.
5845          */
5846         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5847         sgptr &= SG_PTR_MASK;
5848
5849         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5850               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5851               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5852
5853         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5854                 struct ahd_dma64_seg *sg;
5855
5856                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5857
5858                 /* The residual sg_ptr always points to the next sg */
5859                 sg--;
5860
5861                 dataptr = ahd_le64toh(sg->addr)
5862                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5863                         - resid;
5864                 ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5865         } else {
5866                 struct   ahd_dma_seg *sg;
5867
5868                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5869
5870                 /* The residual sg_ptr always points to the next sg */
5871                 sg--;
5872
5873                 dataptr = ahd_le32toh(sg->addr)
5874                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5875                         - resid;
5876                 ahd_outb(ahd, HADDR + 4,
5877                          (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5878         }
5879         ahd_outl(ahd, HADDR, dataptr);
5880         ahd_outb(ahd, HCNT + 2, resid >> 16);
5881         ahd_outb(ahd, HCNT + 1, resid >> 8);
5882         ahd_outb(ahd, HCNT, resid);
5883 }
5884
5885 /*
5886  * Handle the effects of issuing a bus device reset message.
5887  */
5888 static void
5889 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5890                     u_int lun, cam_status status, char *message,
5891                     int verbose_level)
5892 {
5893 #ifdef AHD_TARGET_MODE
5894         struct ahd_tmode_tstate* tstate;
5895 #endif
5896         int found;
5897
5898         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5899                                lun, SCB_LIST_NULL, devinfo->role,
5900                                status);
5901
5902 #ifdef AHD_TARGET_MODE
5903         /*
5904          * Send an immediate notify ccb to all target mord peripheral
5905          * drivers affected by this action.
5906          */
5907         tstate = ahd->enabled_targets[devinfo->our_scsiid];
5908         if (tstate != NULL) {
5909                 u_int cur_lun;
5910                 u_int max_lun;
5911
5912                 if (lun != CAM_LUN_WILDCARD) {
5913                         cur_lun = 0;
5914                         max_lun = AHD_NUM_LUNS - 1;
5915                 } else {
5916                         cur_lun = lun;
5917                         max_lun = lun;
5918                 }
5919                 for (;cur_lun <= max_lun; cur_lun++) {
5920                         struct ahd_tmode_lstate* lstate;
5921
5922                         lstate = tstate->enabled_luns[cur_lun];
5923                         if (lstate == NULL)
5924                                 continue;
5925
5926                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5927                                                MSG_BUS_DEV_RESET, /*arg*/0);
5928                         ahd_send_lstate_events(ahd, lstate);
5929                 }
5930         }
5931 #endif
5932
5933         /*
5934          * Go back to async/narrow transfers and renegotiate.
5935          */
5936         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5937                       AHD_TRANS_CUR, /*paused*/TRUE);
5938         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5939                          /*ppr_options*/0, AHD_TRANS_CUR,
5940                          /*paused*/TRUE);
5941
5942         if (status != CAM_SEL_TIMEOUT)
5943                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
5944                                CAM_LUN_WILDCARD, AC_SENT_BDR);
5945
5946         if (message != NULL && bootverbose)
5947                 printk("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5948                        message, devinfo->channel, devinfo->target, found);
5949 }
5950
5951 #ifdef AHD_TARGET_MODE
5952 static void
5953 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5954                        struct scb *scb)
5955 {
5956
5957         /*
5958          * To facilitate adding multiple messages together,
5959          * each routine should increment the index and len
5960          * variables instead of setting them explicitly.
5961          */
5962         ahd->msgout_index = 0;
5963         ahd->msgout_len = 0;
5964
5965         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
5966                 ahd_build_transfer_msg(ahd, devinfo);
5967         else
5968                 panic("ahd_intr: AWAITING target message with no message");
5969
5970         ahd->msgout_index = 0;
5971         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5972 }
5973 #endif
5974 /**************************** Initialization **********************************/
5975 static u_int
5976 ahd_sglist_size(struct ahd_softc *ahd)
5977 {
5978         bus_size_t list_size;
5979
5980         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
5981         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5982                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
5983         return (list_size);
5984 }
5985
5986 /*
5987  * Calculate the optimum S/G List allocation size.  S/G elements used
5988  * for a given transaction must be physically contiguous.  Assume the
5989  * OS will allocate full pages to us, so it doesn't make sense to request
5990  * less than a page.
5991  */
5992 static u_int
5993 ahd_sglist_allocsize(struct ahd_softc *ahd)
5994 {
5995         bus_size_t sg_list_increment;
5996         bus_size_t sg_list_size;
5997         bus_size_t max_list_size;
5998         bus_size_t best_list_size;
5999
6000         /* Start out with the minimum required for AHD_NSEG. */
6001         sg_list_increment = ahd_sglist_size(ahd);
6002         sg_list_size = sg_list_increment;
6003
6004         /* Get us as close as possible to a page in size. */
6005         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
6006                 sg_list_size += sg_list_increment;
6007
6008         /*
6009          * Try to reduce the amount of wastage by allocating
6010          * multiple pages.
6011          */
6012         best_list_size = sg_list_size;
6013         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
6014         if (max_list_size < 4 * PAGE_SIZE)
6015                 max_list_size = 4 * PAGE_SIZE;
6016         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
6017                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
6018         while ((sg_list_size + sg_list_increment) <= max_list_size
6019            &&  (sg_list_size % PAGE_SIZE) != 0) {
6020                 bus_size_t new_mod;
6021                 bus_size_t best_mod;
6022
6023                 sg_list_size += sg_list_increment;
6024                 new_mod = sg_list_size % PAGE_SIZE;
6025                 best_mod = best_list_size % PAGE_SIZE;
6026                 if (new_mod > best_mod || new_mod == 0) {
6027                         best_list_size = sg_list_size;
6028                 }
6029         }
6030         return (best_list_size);
6031 }
6032
6033 /*
6034  * Allocate a controller structure for a new device
6035  * and perform initial initializion.
6036  */
6037 struct ahd_softc *
6038 ahd_alloc(void *platform_arg, char *name)
6039 {
6040         struct  ahd_softc *ahd;
6041
6042         ahd = kzalloc(sizeof(*ahd), GFP_ATOMIC);
6043         if (!ahd) {
6044                 printk("aic7xxx: cannot malloc softc!\n");
6045                 kfree(name);
6046                 return NULL;
6047         }
6048
6049         ahd->seep_config = kmalloc(sizeof(*ahd->seep_config), GFP_ATOMIC);
6050         if (ahd->seep_config == NULL) {
6051                 kfree(ahd);
6052                 kfree(name);
6053                 return (NULL);
6054         }
6055         LIST_INIT(&ahd->pending_scbs);
6056         /* We don't know our unit number until the OSM sets it */
6057         ahd->name = name;
6058         ahd->unit = -1;
6059         ahd->description = NULL;
6060         ahd->bus_description = NULL;
6061         ahd->channel = 'A';
6062         ahd->chip = AHD_NONE;
6063         ahd->features = AHD_FENONE;
6064         ahd->bugs = AHD_BUGNONE;
6065         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
6066                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
6067         timer_setup(&ahd->stat_timer, ahd_stat_timer, 0);
6068         ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
6069         ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
6070         ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
6071         ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
6072         ahd->int_coalescing_stop_threshold =
6073             AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
6074
6075 #ifdef AHD_DEBUG
6076         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
6077                 printk("%s: scb size = 0x%x, hscb size = 0x%x\n",
6078                        ahd_name(ahd), (u_int)sizeof(struct scb),
6079                        (u_int)sizeof(struct hardware_scb));
6080         }
6081 #endif
6082         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
6083                 ahd_free(ahd);
6084                 ahd = NULL;
6085         }
6086         return (ahd);
6087 }
6088
6089 int
6090 ahd_softc_init(struct ahd_softc *ahd)
6091 {
6092
6093         ahd->unpause = 0;
6094         ahd->pause = PAUSE;
6095         return (0);
6096 }
6097
6098 void
6099 ahd_set_unit(struct ahd_softc *ahd, int unit)
6100 {
6101         ahd->unit = unit;
6102 }
6103
6104 void
6105 ahd_set_name(struct ahd_softc *ahd, char *name)
6106 {
6107         kfree(ahd->name);
6108         ahd->name = name;
6109 }
6110
6111 void
6112 ahd_free(struct ahd_softc *ahd)
6113 {
6114         int i;
6115
6116         switch (ahd->init_level) {
6117         default:
6118         case 5:
6119                 ahd_shutdown(ahd);
6120                 fallthrough;
6121         case 4:
6122                 ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
6123                                   ahd->shared_data_map.dmamap);
6124                 fallthrough;
6125         case 3:
6126                 ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
6127                                 ahd->shared_data_map.dmamap);
6128                 ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
6129                                    ahd->shared_data_map.dmamap);
6130                 fallthrough;
6131         case 2:
6132                 ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
6133         case 1:
6134                 break;
6135         case 0:
6136                 break;
6137         }
6138
6139         ahd_platform_free(ahd);
6140         ahd_fini_scbdata(ahd);
6141         for (i = 0; i < AHD_NUM_TARGETS; i++) {
6142                 struct ahd_tmode_tstate *tstate;
6143
6144                 tstate = ahd->enabled_targets[i];
6145                 if (tstate != NULL) {
6146 #ifdef AHD_TARGET_MODE
6147                         int j;
6148
6149                         for (j = 0; j < AHD_NUM_LUNS; j++) {
6150                                 struct ahd_tmode_lstate *lstate;
6151
6152                                 lstate = tstate->enabled_luns[j];
6153                                 if (lstate != NULL) {
6154                                         xpt_free_path(lstate->path);
6155                                         kfree(lstate);
6156                                 }
6157                         }
6158 #endif
6159                         kfree(tstate);
6160                 }
6161         }
6162 #ifdef AHD_TARGET_MODE
6163         if (ahd->black_hole != NULL) {
6164                 xpt_free_path(ahd->black_hole->path);
6165                 kfree(ahd->black_hole);
6166         }
6167 #endif
6168         kfree(ahd->name);
6169         kfree(ahd->seep_config);
6170         kfree(ahd->saved_stack);
6171         kfree(ahd);
6172         return;
6173 }
6174
6175 static void
6176 ahd_shutdown(void *arg)
6177 {
6178         struct  ahd_softc *ahd;
6179
6180         ahd = (struct ahd_softc *)arg;
6181
6182         /*
6183          * Stop periodic timer callbacks.
6184          */
6185         del_timer_sync(&ahd->stat_timer);
6186
6187         /* This will reset most registers to 0, but not all */
6188         ahd_reset(ahd, /*reinit*/FALSE);
6189 }
6190
6191 /*
6192  * Reset the controller and record some information about it
6193  * that is only available just after a reset.  If "reinit" is
6194  * non-zero, this reset occurred after initial configuration
6195  * and the caller requests that the chip be fully reinitialized
6196  * to a runable state.  Chip interrupts are *not* enabled after
6197  * a reinitialization.  The caller must enable interrupts via
6198  * ahd_intr_enable().
6199  */
6200 int
6201 ahd_reset(struct ahd_softc *ahd, int reinit)
6202 {
6203         u_int    sxfrctl1;
6204         int      wait;
6205         uint32_t cmd;
6206
6207         /*
6208          * Preserve the value of the SXFRCTL1 register for all channels.
6209          * It contains settings that affect termination and we don't want
6210          * to disturb the integrity of the bus.
6211          */
6212         ahd_pause(ahd);
6213         ahd_update_modes(ahd);
6214         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6215         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
6216
6217         cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
6218         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6219                 uint32_t mod_cmd;
6220
6221                 /*
6222                  * A4 Razor #632
6223                  * During the assertion of CHIPRST, the chip
6224                  * does not disable its parity logic prior to
6225                  * the start of the reset.  This may cause a
6226                  * parity error to be detected and thus a
6227                  * spurious SERR or PERR assertion.  Disable
6228                  * PERR and SERR responses during the CHIPRST.
6229                  */
6230                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
6231                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6232                                      mod_cmd, /*bytes*/2);
6233         }
6234         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
6235
6236         /*
6237          * Ensure that the reset has finished.  We delay 1000us
6238          * prior to reading the register to make sure the chip
6239          * has sufficiently completed its reset to handle register
6240          * accesses.
6241          */
6242         wait = 1000;
6243         do {
6244                 ahd_delay(1000);
6245         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
6246
6247         if (wait == 0) {
6248                 printk("%s: WARNING - Failed chip reset!  "
6249                        "Trying to initialize anyway.\n", ahd_name(ahd));
6250         }
6251         ahd_outb(ahd, HCNTRL, ahd->pause);
6252
6253         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6254                 /*
6255                  * Clear any latched PCI error status and restore
6256                  * previous SERR and PERR response enables.
6257                  */
6258                 ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
6259                                      0xFF, /*bytes*/1);
6260                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6261                                      cmd, /*bytes*/2);
6262         }
6263
6264         /*
6265          * Mode should be SCSI after a chip reset, but lets
6266          * set it just to be safe.  We touch the MODE_PTR
6267          * register directly so as to bypass the lazy update
6268          * code in ahd_set_modes().
6269          */
6270         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6271         ahd_outb(ahd, MODE_PTR,
6272                  ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
6273
6274         /*
6275          * Restore SXFRCTL1.
6276          *
6277          * We must always initialize STPWEN to 1 before we
6278          * restore the saved values.  STPWEN is initialized
6279          * to a tri-state condition which can only be cleared
6280          * by turning it on.
6281          */
6282         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
6283         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
6284
6285         /* Determine chip configuration */
6286         ahd->features &= ~AHD_WIDE;
6287         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
6288                 ahd->features |= AHD_WIDE;
6289
6290         /*
6291          * If a recovery action has forced a chip reset,
6292          * re-initialize the chip to our liking.
6293          */
6294         if (reinit != 0)
6295                 ahd_chip_init(ahd);
6296
6297         return (0);
6298 }
6299
6300 /*
6301  * Determine the number of SCBs available on the controller
6302  */
6303 static int
6304 ahd_probe_scbs(struct ahd_softc *ahd) {
6305         int i;
6306
6307         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
6308                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
6309         for (i = 0; i < AHD_SCB_MAX; i++) {
6310                 int j;
6311
6312                 ahd_set_scbptr(ahd, i);
6313                 ahd_outw(ahd, SCB_BASE, i);
6314                 for (j = 2; j < 64; j++)
6315                         ahd_outb(ahd, SCB_BASE+j, 0);
6316                 /* Start out life as unallocated (needing an abort) */
6317                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
6318                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
6319                         break;
6320                 ahd_set_scbptr(ahd, 0);
6321                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
6322                         break;
6323         }
6324         return (i);
6325 }
6326
6327 static void
6328 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
6329 {
6330         dma_addr_t *baddr;
6331
6332         baddr = (dma_addr_t *)arg;
6333         *baddr = segs->ds_addr;
6334 }
6335
6336 static void
6337 ahd_initialize_hscbs(struct ahd_softc *ahd)
6338 {
6339         int i;
6340
6341         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
6342                 ahd_set_scbptr(ahd, i);
6343
6344                 /* Clear the control byte. */
6345                 ahd_outb(ahd, SCB_CONTROL, 0);
6346
6347                 /* Set the next pointer */
6348                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
6349         }
6350 }
6351
6352 static int
6353 ahd_init_scbdata(struct ahd_softc *ahd)
6354 {
6355         struct  scb_data *scb_data;
6356         int     i;
6357
6358         scb_data = &ahd->scb_data;
6359         TAILQ_INIT(&scb_data->free_scbs);
6360         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
6361                 LIST_INIT(&scb_data->free_scb_lists[i]);
6362         LIST_INIT(&scb_data->any_dev_free_scb_list);
6363         SLIST_INIT(&scb_data->hscb_maps);
6364         SLIST_INIT(&scb_data->sg_maps);
6365         SLIST_INIT(&scb_data->sense_maps);
6366
6367         /* Determine the number of hardware SCBs and initialize them */
6368         scb_data->maxhscbs = ahd_probe_scbs(ahd);
6369         if (scb_data->maxhscbs == 0) {
6370                 printk("%s: No SCB space found\n", ahd_name(ahd));
6371                 return (ENXIO);
6372         }
6373
6374         ahd_initialize_hscbs(ahd);
6375
6376         /*
6377          * Create our DMA tags.  These tags define the kinds of device
6378          * accessible memory allocations and memory mappings we will
6379          * need to perform during normal operation.
6380          *
6381          * Unless we need to further restrict the allocation, we rely
6382          * on the restrictions of the parent dmat, hence the common
6383          * use of MAXADDR and MAXSIZE.
6384          */
6385
6386         /* DMA tag for our hardware scb structures */
6387         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6388                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6389                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6390                                /*highaddr*/BUS_SPACE_MAXADDR,
6391                                /*filter*/NULL, /*filterarg*/NULL,
6392                                PAGE_SIZE, /*nsegments*/1,
6393                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6394                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
6395                 goto error_exit;
6396         }
6397
6398         scb_data->init_level++;
6399
6400         /* DMA tag for our S/G structures. */
6401         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
6402                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6403                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6404                                /*highaddr*/BUS_SPACE_MAXADDR,
6405                                /*filter*/NULL, /*filterarg*/NULL,
6406                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
6407                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6408                                /*flags*/0, &scb_data->sg_dmat) != 0) {
6409                 goto error_exit;
6410         }
6411 #ifdef AHD_DEBUG
6412         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
6413                 printk("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
6414                        ahd_sglist_allocsize(ahd));
6415 #endif
6416
6417         scb_data->init_level++;
6418
6419         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
6420         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6421                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6422                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6423                                /*highaddr*/BUS_SPACE_MAXADDR,
6424                                /*filter*/NULL, /*filterarg*/NULL,
6425                                PAGE_SIZE, /*nsegments*/1,
6426                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6427                                /*flags*/0, &scb_data->sense_dmat) != 0) {
6428                 goto error_exit;
6429         }
6430
6431         scb_data->init_level++;
6432
6433         /* Perform initial CCB allocation */
6434         ahd_alloc_scbs(ahd);
6435
6436         if (scb_data->numscbs == 0) {
6437                 printk("%s: ahd_init_scbdata - "
6438                        "Unable to allocate initial scbs\n",
6439                        ahd_name(ahd));
6440                 goto error_exit;
6441         }
6442
6443         /*
6444          * Note that we were successful
6445          */
6446         return (0);
6447
6448 error_exit:
6449
6450         return (ENOMEM);
6451 }
6452
6453 static struct scb *
6454 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
6455 {
6456         struct scb *scb;
6457
6458         /*
6459          * Look on the pending list.
6460          */
6461         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
6462                 if (SCB_GET_TAG(scb) == tag)
6463                         return (scb);
6464         }
6465
6466         /*
6467          * Then on all of the collision free lists.
6468          */
6469         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6470                 struct scb *list_scb;
6471
6472                 list_scb = scb;
6473                 do {
6474                         if (SCB_GET_TAG(list_scb) == tag)
6475                                 return (list_scb);
6476                         list_scb = LIST_NEXT(list_scb, collision_links);
6477                 } while (list_scb);
6478         }
6479
6480         /*
6481          * And finally on the generic free list.
6482          */
6483         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
6484                 if (SCB_GET_TAG(scb) == tag)
6485                         return (scb);
6486         }
6487
6488         return (NULL);
6489 }
6490
6491 static void
6492 ahd_fini_scbdata(struct ahd_softc *ahd)
6493 {
6494         struct scb_data *scb_data;
6495
6496         scb_data = &ahd->scb_data;
6497         if (scb_data == NULL)
6498                 return;
6499
6500         switch (scb_data->init_level) {
6501         default:
6502         case 7:
6503         {
6504                 struct map_node *sns_map;
6505
6506                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
6507                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
6508                         ahd_dmamap_unload(ahd, scb_data->sense_dmat,
6509                                           sns_map->dmamap);
6510                         ahd_dmamem_free(ahd, scb_data->sense_dmat,
6511                                         sns_map->vaddr, sns_map->dmamap);
6512                         kfree(sns_map);
6513                 }
6514                 ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
6515         }
6516                 fallthrough;
6517         case 6:
6518         {
6519                 struct map_node *sg_map;
6520
6521                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
6522                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
6523                         ahd_dmamap_unload(ahd, scb_data->sg_dmat,
6524                                           sg_map->dmamap);
6525                         ahd_dmamem_free(ahd, scb_data->sg_dmat,
6526                                         sg_map->vaddr, sg_map->dmamap);
6527                         kfree(sg_map);
6528                 }
6529                 ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
6530         }
6531                 fallthrough;
6532         case 5:
6533         {
6534                 struct map_node *hscb_map;
6535
6536                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
6537                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
6538                         ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
6539                                           hscb_map->dmamap);
6540                         ahd_dmamem_free(ahd, scb_data->hscb_dmat,
6541                                         hscb_map->vaddr, hscb_map->dmamap);
6542                         kfree(hscb_map);
6543                 }
6544                 ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
6545                 /* FALLTHROUGH */
6546         }
6547         case 4:
6548         case 3:
6549         case 2:
6550         case 1:
6551         case 0:
6552                 break;
6553         }
6554 }
6555
6556 /*
6557  * DSP filter Bypass must be enabled until the first selection
6558  * after a change in bus mode (Razor #491 and #493).
6559  */
6560 static void
6561 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
6562 {
6563         ahd_mode_state saved_modes;
6564
6565         saved_modes = ahd_save_modes(ahd);
6566         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6567         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
6568                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
6569         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
6570 #ifdef AHD_DEBUG
6571         if ((ahd_debug & AHD_SHOW_MISC) != 0)
6572                 printk("%s: Setting up iocell workaround\n", ahd_name(ahd));
6573 #endif
6574         ahd_restore_modes(ahd, saved_modes);
6575         ahd->flags &= ~AHD_HAD_FIRST_SEL;
6576 }
6577
6578 static void
6579 ahd_iocell_first_selection(struct ahd_softc *ahd)
6580 {
6581         ahd_mode_state  saved_modes;
6582         u_int           sblkctl;
6583
6584         if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
6585                 return;
6586         saved_modes = ahd_save_modes(ahd);
6587         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6588         sblkctl = ahd_inb(ahd, SBLKCTL);
6589         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6590 #ifdef AHD_DEBUG
6591         if ((ahd_debug & AHD_SHOW_MISC) != 0)
6592                 printk("%s: iocell first selection\n", ahd_name(ahd));
6593 #endif
6594         if ((sblkctl & ENAB40) != 0) {
6595                 ahd_outb(ahd, DSPDATACTL,
6596                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
6597 #ifdef AHD_DEBUG
6598                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6599                         printk("%s: BYPASS now disabled\n", ahd_name(ahd));
6600 #endif
6601         }
6602         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
6603         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6604         ahd_restore_modes(ahd, saved_modes);
6605         ahd->flags |= AHD_HAD_FIRST_SEL;
6606 }
6607
6608 /*************************** SCB Management ***********************************/
6609 static void
6610 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
6611 {
6612         struct  scb_list *free_list;
6613         struct  scb_tailq *free_tailq;
6614         struct  scb *first_scb;
6615
6616         scb->flags |= SCB_ON_COL_LIST;
6617         AHD_SET_SCB_COL_IDX(scb, col_idx);
6618         free_list = &ahd->scb_data.free_scb_lists[col_idx];
6619         free_tailq = &ahd->scb_data.free_scbs;
6620         first_scb = LIST_FIRST(free_list);
6621         if (first_scb != NULL) {
6622                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
6623         } else {
6624                 LIST_INSERT_HEAD(free_list, scb, collision_links);
6625                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
6626         }
6627 }
6628
6629 static void
6630 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
6631 {
6632         struct  scb_list *free_list;
6633         struct  scb_tailq *free_tailq;
6634         struct  scb *first_scb;
6635         u_int   col_idx;
6636
6637         scb->flags &= ~SCB_ON_COL_LIST;
6638         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
6639         free_list = &ahd->scb_data.free_scb_lists[col_idx];
6640         free_tailq = &ahd->scb_data.free_scbs;
6641         first_scb = LIST_FIRST(free_list);
6642         if (first_scb == scb) {
6643                 struct scb *next_scb;
6644
6645                 /*
6646                  * Maintain order in the collision free
6647                  * lists for fairness if this device has
6648                  * other colliding tags active.
6649                  */
6650                 next_scb = LIST_NEXT(scb, collision_links);
6651                 if (next_scb != NULL) {
6652                         TAILQ_INSERT_AFTER(free_tailq, scb,
6653                                            next_scb, links.tqe);
6654                 }
6655                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
6656         }
6657         LIST_REMOVE(scb, collision_links);
6658 }
6659
6660 /*
6661  * Get a free scb. If there are none, see if we can allocate a new SCB.
6662  */
6663 struct scb *
6664 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
6665 {
6666         struct scb *scb;
6667         int tries;
6668
6669         tries = 0;
6670 look_again:
6671         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6672                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
6673                         ahd_rem_col_list(ahd, scb);
6674                         goto found;
6675                 }
6676         }
6677         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
6678
6679                 if (tries++ != 0)
6680                         return (NULL);
6681                 ahd_alloc_scbs(ahd);
6682                 goto look_again;
6683         }
6684         LIST_REMOVE(scb, links.le);
6685         if (col_idx != AHD_NEVER_COL_IDX
6686          && (scb->col_scb != NULL)
6687          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
6688                 LIST_REMOVE(scb->col_scb, links.le);
6689                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
6690         }
6691 found:
6692         scb->flags |= SCB_ACTIVE;
6693         return (scb);
6694 }
6695
6696 /*
6697  * Return an SCB resource to the free list.
6698  */
6699 void
6700 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
6701 {
6702         /* Clean up for the next user */
6703         scb->flags = SCB_FLAG_NONE;
6704         scb->hscb->control = 0;
6705         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
6706
6707         if (scb->col_scb == NULL) {
6708
6709                 /*
6710                  * No collision possible.  Just free normally.
6711                  */
6712                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6713                                  scb, links.le);
6714         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
6715
6716                 /*
6717                  * The SCB we might have collided with is on
6718                  * a free collision list.  Put both SCBs on
6719                  * the generic list.
6720                  */
6721                 ahd_rem_col_list(ahd, scb->col_scb);
6722                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6723                                  scb, links.le);
6724                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6725                                  scb->col_scb, links.le);
6726         } else if ((scb->col_scb->flags
6727                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
6728                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
6729
6730                 /*
6731                  * The SCB we might collide with on the next allocation
6732                  * is still active in a non-packetized, tagged, context.
6733                  * Put us on the SCB collision list.
6734                  */
6735                 ahd_add_col_list(ahd, scb,
6736                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
6737         } else {
6738                 /*
6739                  * The SCB we might collide with on the next allocation
6740                  * is either active in a packetized context, or free.
6741                  * Since we can't collide, put this SCB on the generic
6742                  * free list.
6743                  */
6744                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6745                                  scb, links.le);
6746         }
6747
6748         ahd_platform_scb_free(ahd, scb);
6749 }
6750
6751 static void
6752 ahd_alloc_scbs(struct ahd_softc *ahd)
6753 {
6754         struct scb_data *scb_data;
6755         struct scb      *next_scb;
6756         struct hardware_scb *hscb;
6757         struct map_node *hscb_map;
6758         struct map_node *sg_map;
6759         struct map_node *sense_map;
6760         uint8_t         *segs;
6761         uint8_t         *sense_data;
6762         dma_addr_t       hscb_busaddr;
6763         dma_addr_t       sg_busaddr;
6764         dma_addr_t       sense_busaddr;
6765         int              newcount;
6766         int              i;
6767
6768         scb_data = &ahd->scb_data;
6769         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
6770                 /* Can't allocate any more */
6771                 return;
6772
6773         if (scb_data->scbs_left != 0) {
6774                 int offset;
6775
6776                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
6777                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
6778                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
6779                 hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
6780         } else {
6781                 hscb_map = kmalloc(sizeof(*hscb_map), GFP_ATOMIC);
6782
6783                 if (hscb_map == NULL)
6784                         return;
6785
6786                 /* Allocate the next batch of hardware SCBs */
6787                 if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
6788                                      (void **)&hscb_map->vaddr,
6789                                      BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
6790                         kfree(hscb_map);
6791                         return;
6792                 }
6793
6794                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6795
6796                 ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6797                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6798                                 &hscb_map->physaddr, /*flags*/0);
6799
6800                 hscb = (struct hardware_scb *)hscb_map->vaddr;
6801                 hscb_busaddr = hscb_map->physaddr;
6802                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6803         }
6804
6805         if (scb_data->sgs_left != 0) {
6806                 int offset;
6807
6808                 offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6809                        - scb_data->sgs_left) * ahd_sglist_size(ahd);
6810                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
6811                 segs = sg_map->vaddr + offset;
6812                 sg_busaddr = sg_map->physaddr + offset;
6813         } else {
6814                 sg_map = kmalloc(sizeof(*sg_map), GFP_ATOMIC);
6815
6816                 if (sg_map == NULL)
6817                         return;
6818
6819                 /* Allocate the next batch of S/G lists */
6820                 if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
6821                                      (void **)&sg_map->vaddr,
6822                                      BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
6823                         kfree(sg_map);
6824                         return;
6825                 }
6826
6827                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6828
6829                 ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6830                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
6831                                 ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
6832
6833                 segs = sg_map->vaddr;
6834                 sg_busaddr = sg_map->physaddr;
6835                 scb_data->sgs_left =
6836                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6837 #ifdef AHD_DEBUG
6838                 if (ahd_debug & AHD_SHOW_MEMORY)
6839                         printk("Mapped SG data\n");
6840 #endif
6841         }
6842
6843         if (scb_data->sense_left != 0) {
6844                 int offset;
6845
6846                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6847                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
6848                 sense_data = sense_map->vaddr + offset;
6849                 sense_busaddr = sense_map->physaddr + offset;
6850         } else {
6851                 sense_map = kmalloc(sizeof(*sense_map), GFP_ATOMIC);
6852
6853                 if (sense_map == NULL)
6854                         return;
6855
6856                 /* Allocate the next batch of sense buffers */
6857                 if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
6858                                      (void **)&sense_map->vaddr,
6859                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6860                         kfree(sense_map);
6861                         return;
6862                 }
6863
6864                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6865
6866                 ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6867                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6868                                 &sense_map->physaddr, /*flags*/0);
6869
6870                 sense_data = sense_map->vaddr;
6871                 sense_busaddr = sense_map->physaddr;
6872                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6873 #ifdef AHD_DEBUG
6874                 if (ahd_debug & AHD_SHOW_MEMORY)
6875                         printk("Mapped sense data\n");
6876 #endif
6877         }
6878
6879         newcount = min(scb_data->sense_left, scb_data->scbs_left);
6880         newcount = min(newcount, scb_data->sgs_left);
6881         newcount = min(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6882         for (i = 0; i < newcount; i++) {
6883                 struct scb_platform_data *pdata;
6884                 u_int col_tag;
6885
6886                 next_scb = kmalloc(sizeof(*next_scb), GFP_ATOMIC);
6887                 if (next_scb == NULL)
6888                         break;
6889
6890                 pdata = kmalloc(sizeof(*pdata), GFP_ATOMIC);
6891                 if (pdata == NULL) {
6892                         kfree(next_scb);
6893                         break;
6894                 }
6895                 next_scb->platform_data = pdata;
6896                 next_scb->hscb_map = hscb_map;
6897                 next_scb->sg_map = sg_map;
6898                 next_scb->sense_map = sense_map;
6899                 next_scb->sg_list = segs;
6900                 next_scb->sense_data = sense_data;
6901                 next_scb->sense_busaddr = sense_busaddr;
6902                 memset(hscb, 0, sizeof(*hscb));
6903                 next_scb->hscb = hscb;
6904                 hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
6905
6906                 /*
6907                  * The sequencer always starts with the second entry.
6908                  * The first entry is embedded in the scb.
6909                  */
6910                 next_scb->sg_list_busaddr = sg_busaddr;
6911                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6912                         next_scb->sg_list_busaddr
6913                             += sizeof(struct ahd_dma64_seg);
6914                 else
6915                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6916                 next_scb->ahd_softc = ahd;
6917                 next_scb->flags = SCB_FLAG_NONE;
6918                 next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
6919                 col_tag = scb_data->numscbs ^ 0x100;
6920                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6921                 if (next_scb->col_scb != NULL)
6922                         next_scb->col_scb->col_scb = next_scb;
6923                 ahd_free_scb(ahd, next_scb);
6924                 hscb++;
6925                 hscb_busaddr += sizeof(*hscb);
6926                 segs += ahd_sglist_size(ahd);
6927                 sg_busaddr += ahd_sglist_size(ahd);
6928                 sense_data += AHD_SENSE_BUFSIZE;
6929                 sense_busaddr += AHD_SENSE_BUFSIZE;
6930                 scb_data->numscbs++;
6931                 scb_data->sense_left--;
6932                 scb_data->scbs_left--;
6933                 scb_data->sgs_left--;
6934         }
6935 }
6936
6937 void
6938 ahd_controller_info(struct ahd_softc *ahd, char *buf)
6939 {
6940         const char *speed;
6941         const char *type;
6942         int len;
6943
6944         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
6945         buf += len;
6946
6947         speed = "Ultra320 ";
6948         if ((ahd->features & AHD_WIDE) != 0) {
6949                 type = "Wide ";
6950         } else {
6951                 type = "Single ";
6952         }
6953         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
6954                       speed, type, ahd->channel, ahd->our_id);
6955         buf += len;
6956
6957         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
6958                 ahd->scb_data.maxhscbs);
6959 }
6960
6961 static const char *channel_strings[] = {
6962         "Primary Low",
6963         "Primary High",
6964         "Secondary Low",
6965         "Secondary High"
6966 };
6967
6968 static const char *termstat_strings[] = {
6969         "Terminated Correctly",
6970         "Over Terminated",
6971         "Under Terminated",
6972         "Not Configured"
6973 };
6974
6975 /***************************** Timer Facilities *******************************/
6976 static void
6977 ahd_timer_reset(struct timer_list *timer, int usec)
6978 {
6979         del_timer(timer);
6980         timer->expires = jiffies + (usec * HZ)/1000000;
6981         add_timer(timer);
6982 }
6983
6984 /*
6985  * Start the board, ready for normal operation
6986  */
6987 int
6988 ahd_init(struct ahd_softc *ahd)
6989 {
6990         uint8_t         *next_vaddr;
6991         dma_addr_t       next_baddr;
6992         size_t           driver_data_size;
6993         int              i;
6994         int              error;
6995         u_int            warn_user;
6996         uint8_t          current_sensing;
6997         uint8_t          fstat;
6998
6999         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7000
7001         ahd->stack_size = ahd_probe_stack_size(ahd);
7002         ahd->saved_stack = kmalloc_array(ahd->stack_size, sizeof(uint16_t),
7003                                          GFP_ATOMIC);
7004         if (ahd->saved_stack == NULL)
7005                 return (ENOMEM);
7006
7007         /*
7008          * Verify that the compiler hasn't over-aggressively
7009          * padded important structures.
7010          */
7011         if (sizeof(struct hardware_scb) != 64)
7012                 panic("Hardware SCB size is incorrect");
7013
7014 #ifdef AHD_DEBUG
7015         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
7016                 ahd->flags |= AHD_SEQUENCER_DEBUG;
7017 #endif
7018
7019         /*
7020          * Default to allowing initiator operations.
7021          */
7022         ahd->flags |= AHD_INITIATORROLE;
7023
7024         /*
7025          * Only allow target mode features if this unit has them enabled.
7026          */
7027         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
7028                 ahd->features &= ~AHD_TARGETMODE;
7029
7030         ahd->init_level++;
7031
7032         /*
7033          * DMA tag for our command fifos and other data in system memory
7034          * the card's sequencer must be able to access.  For initiator
7035          * roles, we need to allocate space for the qoutfifo.  When providing
7036          * for the target mode role, we must additionally provide space for
7037          * the incoming target command fifo.
7038          */
7039         driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
7040                          + sizeof(struct hardware_scb);
7041         if ((ahd->features & AHD_TARGETMODE) != 0)
7042                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7043         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
7044                 driver_data_size += PKT_OVERRUN_BUFSIZE;
7045         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
7046                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
7047                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
7048                                /*highaddr*/BUS_SPACE_MAXADDR,
7049                                /*filter*/NULL, /*filterarg*/NULL,
7050                                driver_data_size,
7051                                /*nsegments*/1,
7052                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
7053                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
7054                 return (ENOMEM);
7055         }
7056
7057         ahd->init_level++;
7058
7059         /* Allocation of driver data */
7060         if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
7061                              (void **)&ahd->shared_data_map.vaddr,
7062                              BUS_DMA_NOWAIT,
7063                              &ahd->shared_data_map.dmamap) != 0) {
7064                 return (ENOMEM);
7065         }
7066
7067         ahd->init_level++;
7068
7069         /* And permanently map it in */
7070         ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
7071                         ahd->shared_data_map.vaddr, driver_data_size,
7072                         ahd_dmamap_cb, &ahd->shared_data_map.physaddr,
7073                         /*flags*/0);
7074         ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
7075         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
7076         next_baddr = ahd->shared_data_map.physaddr
7077                    + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
7078         if ((ahd->features & AHD_TARGETMODE) != 0) {
7079                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
7080                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7081                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7082         }
7083
7084         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
7085                 ahd->overrun_buf = next_vaddr;
7086                 next_vaddr += PKT_OVERRUN_BUFSIZE;
7087                 next_baddr += PKT_OVERRUN_BUFSIZE;
7088         }
7089
7090         /*
7091          * We need one SCB to serve as the "next SCB".  Since the
7092          * tag identifier in this SCB will never be used, there is
7093          * no point in using a valid HSCB tag from an SCB pulled from
7094          * the standard free pool.  So, we allocate this "sentinel"
7095          * specially from the DMA safe memory chunk used for the QOUTFIFO.
7096          */
7097         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
7098         ahd->next_queued_hscb_map = &ahd->shared_data_map;
7099         ahd->next_queued_hscb->hscb_busaddr = ahd_htole32(next_baddr);
7100
7101         ahd->init_level++;
7102
7103         /* Allocate SCB data now that buffer_dmat is initialized */
7104         if (ahd_init_scbdata(ahd) != 0)
7105                 return (ENOMEM);
7106
7107         if ((ahd->flags & AHD_INITIATORROLE) == 0)
7108                 ahd->flags &= ~AHD_RESET_BUS_A;
7109
7110         /*
7111          * Before committing these settings to the chip, give
7112          * the OSM one last chance to modify our configuration.
7113          */
7114         ahd_platform_init(ahd);
7115
7116         /* Bring up the chip. */
7117         ahd_chip_init(ahd);
7118
7119         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7120
7121         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
7122                 goto init_done;
7123
7124         /*
7125          * Verify termination based on current draw and
7126          * warn user if the bus is over/under terminated.
7127          */
7128         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
7129                                    CURSENSE_ENB);
7130         if (error != 0) {
7131                 printk("%s: current sensing timeout 1\n", ahd_name(ahd));
7132                 goto init_done;
7133         }
7134         for (i = 20, fstat = FLX_FSTAT_BUSY;
7135              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
7136                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
7137                 if (error != 0) {
7138                         printk("%s: current sensing timeout 2\n",
7139                                ahd_name(ahd));
7140                         goto init_done;
7141                 }
7142         }
7143         if (i == 0) {
7144                 printk("%s: Timedout during current-sensing test\n",
7145                        ahd_name(ahd));
7146                 goto init_done;
7147         }
7148
7149         /* Latch Current Sensing status. */
7150         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
7151         if (error != 0) {
7152                 printk("%s: current sensing timeout 3\n", ahd_name(ahd));
7153                 goto init_done;
7154         }
7155
7156         /* Diable current sensing. */
7157         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
7158
7159 #ifdef AHD_DEBUG
7160         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
7161                 printk("%s: current_sensing == 0x%x\n",
7162                        ahd_name(ahd), current_sensing);
7163         }
7164 #endif
7165         warn_user = 0;
7166         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
7167                 u_int term_stat;
7168
7169                 term_stat = (current_sensing & FLX_CSTAT_MASK);
7170                 switch (term_stat) {
7171                 case FLX_CSTAT_OVER:
7172                 case FLX_CSTAT_UNDER:
7173                         warn_user++;
7174                         fallthrough;
7175                 case FLX_CSTAT_INVALID:
7176                 case FLX_CSTAT_OKAY:
7177                         if (warn_user == 0 && bootverbose == 0)
7178                                 break;
7179                         printk("%s: %s Channel %s\n", ahd_name(ahd),
7180                                channel_strings[i], termstat_strings[term_stat]);
7181                         break;
7182                 }
7183         }
7184         if (warn_user) {
7185                 printk("%s: WARNING. Termination is not configured correctly.\n"
7186                        "%s: WARNING. SCSI bus operations may FAIL.\n",
7187                        ahd_name(ahd), ahd_name(ahd));
7188         }
7189 init_done:
7190         ahd_restart(ahd);
7191         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
7192         return (0);
7193 }
7194
7195 /*
7196  * (Re)initialize chip state after a chip reset.
7197  */
7198 static void
7199 ahd_chip_init(struct ahd_softc *ahd)
7200 {
7201         uint32_t busaddr;
7202         u_int    sxfrctl1;
7203         u_int    scsiseq_template;
7204         u_int    wait;
7205         u_int    i;
7206         u_int    target;
7207
7208         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7209         /*
7210          * Take the LED out of diagnostic mode
7211          */
7212         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
7213
7214         /*
7215          * Return HS_MAILBOX to its default value.
7216          */
7217         ahd->hs_mailbox = 0;
7218         ahd_outb(ahd, HS_MAILBOX, 0);
7219
7220         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
7221         ahd_outb(ahd, IOWNID, ahd->our_id);
7222         ahd_outb(ahd, TOWNID, ahd->our_id);
7223         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
7224         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
7225         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
7226          && (ahd->seltime != STIMESEL_MIN)) {
7227                 /*
7228                  * The selection timer duration is twice as long
7229                  * as it should be.  Halve it by adding "1" to
7230                  * the user specified setting.
7231                  */
7232                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
7233         } else {
7234                 sxfrctl1 |= ahd->seltime;
7235         }
7236
7237         ahd_outb(ahd, SXFRCTL0, DFON);
7238         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
7239         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
7240
7241         /*
7242          * Now that termination is set, wait for up
7243          * to 500ms for our transceivers to settle.  If
7244          * the adapter does not have a cable attached,
7245          * the transceivers may never settle, so don't
7246          * complain if we fail here.
7247          */
7248         for (wait = 10000;
7249              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
7250              wait--)
7251                 ahd_delay(100);
7252
7253         /* Clear any false bus resets due to the transceivers settling */
7254         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
7255         ahd_outb(ahd, CLRINT, CLRSCSIINT);
7256
7257         /* Initialize mode specific S/G state. */
7258         for (i = 0; i < 2; i++) {
7259                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
7260                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
7261                 ahd_outb(ahd, SG_STATE, 0);
7262                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
7263                 ahd_outb(ahd, SEQIMODE,
7264                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
7265                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
7266         }
7267
7268         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
7269         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
7270         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
7271         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
7272         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
7273         if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
7274                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
7275         } else {
7276                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
7277         }
7278         ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
7279         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
7280                 /*
7281                  * Do not issue a target abort when a split completion
7282                  * error occurs.  Let our PCIX interrupt handler deal
7283                  * with it instead. H2A4 Razor #625
7284                  */
7285                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
7286
7287         if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
7288                 ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
7289
7290         /*
7291          * Tweak IOCELL settings.
7292          */
7293         if ((ahd->flags & AHD_HP_BOARD) != 0) {
7294                 for (i = 0; i < NUMDSPS; i++) {
7295                         ahd_outb(ahd, DSPSELECT, i);
7296                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
7297                 }
7298 #ifdef AHD_DEBUG
7299                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7300                         printk("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
7301                                WRTBIASCTL_HP_DEFAULT);
7302 #endif
7303         }
7304         ahd_setup_iocell_workaround(ahd);
7305
7306         /*
7307          * Enable LQI Manager interrupts.
7308          */
7309         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
7310                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
7311                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
7312         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
7313         /*
7314          * We choose to have the sequencer catch LQOPHCHGINPKT errors
7315          * manually for the command phase at the start of a packetized
7316          * selection case.  ENLQOBUSFREE should be made redundant by
7317          * the BUSFREE interrupt, but it seems that some LQOBUSFREE
7318          * events fail to assert the BUSFREE interrupt so we must
7319          * also enable LQOBUSFREE interrupts.
7320          */
7321         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
7322
7323         /*
7324          * Setup sequencer interrupt handlers.
7325          */
7326         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
7327         ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
7328
7329         /*
7330          * Setup SCB Offset registers.
7331          */
7332         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7333                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
7334                          pkt_long_lun));
7335         } else {
7336                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
7337         }
7338         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
7339         ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
7340         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
7341         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
7342                                        shared_data.idata.cdb));
7343         ahd_outb(ahd, QNEXTPTR,
7344                  offsetof(struct hardware_scb, next_hscb_busaddr));
7345         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
7346         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
7347         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7348                 ahd_outb(ahd, LUNLEN,
7349                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
7350         } else {
7351                 ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
7352         }
7353         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
7354         ahd_outb(ahd, MAXCMD, 0xFF);
7355         ahd_outb(ahd, SCBAUTOPTR,
7356                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
7357
7358         /* We haven't been enabled for target mode yet. */
7359         ahd_outb(ahd, MULTARGID, 0);
7360         ahd_outb(ahd, MULTARGID + 1, 0);
7361
7362         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7363         /* Initialize the negotiation table. */
7364         if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
7365                 /*
7366                  * Clear the spare bytes in the neg table to avoid
7367                  * spurious parity errors.
7368                  */
7369                 for (target = 0; target < AHD_NUM_TARGETS; target++) {
7370                         ahd_outb(ahd, NEGOADDR, target);
7371                         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
7372                         for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
7373                                 ahd_outb(ahd, ANNEXDAT, 0);
7374                 }
7375         }
7376         for (target = 0; target < AHD_NUM_TARGETS; target++) {
7377                 struct   ahd_devinfo devinfo;
7378                 struct   ahd_initiator_tinfo *tinfo;
7379                 struct   ahd_tmode_tstate *tstate;
7380
7381                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7382                                             target, &tstate);
7383                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7384                                     target, CAM_LUN_WILDCARD,
7385                                     'A', ROLE_INITIATOR);
7386                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
7387         }
7388
7389         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
7390         ahd_outb(ahd, CLRINT, CLRSCSIINT);
7391
7392 #ifdef NEEDS_MORE_TESTING
7393         /*
7394          * Always enable abort on incoming L_Qs if this feature is
7395          * supported.  We use this to catch invalid SCB references.
7396          */
7397         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
7398                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
7399         else
7400 #endif
7401                 ahd_outb(ahd, LQCTL1, 0);
7402
7403         /* All of our queues are empty */
7404         ahd->qoutfifonext = 0;
7405         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
7406         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
7407         for (i = 0; i < AHD_QOUT_SIZE; i++)
7408                 ahd->qoutfifo[i].valid_tag = 0;
7409         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
7410
7411         ahd->qinfifonext = 0;
7412         for (i = 0; i < AHD_QIN_SIZE; i++)
7413                 ahd->qinfifo[i] = SCB_LIST_NULL;
7414
7415         if ((ahd->features & AHD_TARGETMODE) != 0) {
7416                 /* All target command blocks start out invalid. */
7417                 for (i = 0; i < AHD_TMODE_CMDS; i++)
7418                         ahd->targetcmds[i].cmd_valid = 0;
7419                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
7420                 ahd->tqinfifonext = 1;
7421                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
7422                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
7423         }
7424
7425         /* Initialize Scratch Ram. */
7426         ahd_outb(ahd, SEQ_FLAGS, 0);
7427         ahd_outb(ahd, SEQ_FLAGS2, 0);
7428
7429         /* We don't have any waiting selections */
7430         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
7431         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
7432         ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
7433         ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
7434         for (i = 0; i < AHD_NUM_TARGETS; i++)
7435                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
7436
7437         /*
7438          * Nobody is waiting to be DMAed into the QOUTFIFO.
7439          */
7440         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
7441         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
7442         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
7443         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
7444         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
7445
7446         /*
7447          * The Freeze Count is 0.
7448          */
7449         ahd->qfreeze_cnt = 0;
7450         ahd_outw(ahd, QFREEZE_COUNT, 0);
7451         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
7452
7453         /*
7454          * Tell the sequencer where it can find our arrays in memory.
7455          */
7456         busaddr = ahd->shared_data_map.physaddr;
7457         ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
7458         ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
7459
7460         /*
7461          * Setup the allowed SCSI Sequences based on operational mode.
7462          * If we are a target, we'll enable select in operations once
7463          * we've had a lun enabled.
7464          */
7465         scsiseq_template = ENAUTOATNP;
7466         if ((ahd->flags & AHD_INITIATORROLE) != 0)
7467                 scsiseq_template |= ENRSELI;
7468         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
7469
7470         /* There are no busy SCBs yet. */
7471         for (target = 0; target < AHD_NUM_TARGETS; target++) {
7472                 int lun;
7473
7474                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
7475                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
7476         }
7477
7478         /*
7479          * Initialize the group code to command length table.
7480          * Vendor Unique codes are set to 0 so we only capture
7481          * the first byte of the cdb.  These can be overridden
7482          * when target mode is enabled.
7483          */
7484         ahd_outb(ahd, CMDSIZE_TABLE, 5);
7485         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
7486         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
7487         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
7488         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
7489         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
7490         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
7491         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
7492
7493         /* Tell the sequencer of our initial queue positions */
7494         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7495         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
7496         ahd->qinfifonext = 0;
7497         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7498         ahd_set_hescb_qoff(ahd, 0);
7499         ahd_set_snscb_qoff(ahd, 0);
7500         ahd_set_sescb_qoff(ahd, 0);
7501         ahd_set_sdscb_qoff(ahd, 0);
7502
7503         /*
7504          * Tell the sequencer which SCB will be the next one it receives.
7505          */
7506         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7507         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7508
7509         /*
7510          * Default to coalescing disabled.
7511          */
7512         ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
7513         ahd_outw(ahd, CMDS_PENDING, 0);
7514         ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
7515                                      ahd->int_coalescing_maxcmds,
7516                                      ahd->int_coalescing_mincmds);
7517         ahd_enable_coalescing(ahd, FALSE);
7518
7519         ahd_loadseq(ahd);
7520         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7521
7522         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
7523                 u_int negodat3 = ahd_inb(ahd, NEGCONOPTS);
7524
7525                 negodat3 |= ENSLOWCRC;
7526                 ahd_outb(ahd, NEGCONOPTS, negodat3);
7527                 negodat3 = ahd_inb(ahd, NEGCONOPTS);
7528                 if (!(negodat3 & ENSLOWCRC))
7529                         printk("aic79xx: failed to set the SLOWCRC bit\n");
7530                 else
7531                         printk("aic79xx: SLOWCRC bit set\n");
7532         }
7533 }
7534
7535 /*
7536  * Setup default device and controller settings.
7537  * This should only be called if our probe has
7538  * determined that no configuration data is available.
7539  */
7540 int
7541 ahd_default_config(struct ahd_softc *ahd)
7542 {
7543         int     targ;
7544
7545         ahd->our_id = 7;
7546
7547         /*
7548          * Allocate a tstate to house information for our
7549          * initiator presence on the bus as well as the user
7550          * data for any target mode initiator.
7551          */
7552         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7553                 printk("%s: unable to allocate ahd_tmode_tstate.  "
7554                        "Failing attach\n", ahd_name(ahd));
7555                 return (ENOMEM);
7556         }
7557
7558         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
7559                 struct   ahd_devinfo devinfo;
7560                 struct   ahd_initiator_tinfo *tinfo;
7561                 struct   ahd_tmode_tstate *tstate;
7562                 uint16_t target_mask;
7563
7564                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7565                                             targ, &tstate);
7566                 /*
7567                  * We support SPC2 and SPI4.
7568                  */
7569                 tinfo->user.protocol_version = 4;
7570                 tinfo->user.transport_version = 4;
7571
7572                 target_mask = 0x01 << targ;
7573                 ahd->user_discenable |= target_mask;
7574                 tstate->discenable |= target_mask;
7575                 ahd->user_tagenable |= target_mask;
7576 #ifdef AHD_FORCE_160
7577                 tinfo->user.period = AHD_SYNCRATE_DT;
7578 #else
7579                 tinfo->user.period = AHD_SYNCRATE_160;
7580 #endif
7581                 tinfo->user.offset = MAX_OFFSET;
7582                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
7583                                         | MSG_EXT_PPR_WR_FLOW
7584                                         | MSG_EXT_PPR_HOLD_MCS
7585                                         | MSG_EXT_PPR_IU_REQ
7586                                         | MSG_EXT_PPR_QAS_REQ
7587                                         | MSG_EXT_PPR_DT_REQ;
7588                 if ((ahd->features & AHD_RTI) != 0)
7589                         tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
7590
7591                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
7592
7593                 /*
7594                  * Start out Async/Narrow/Untagged and with
7595                  * conservative protocol support.
7596                  */
7597                 tinfo->goal.protocol_version = 2;
7598                 tinfo->goal.transport_version = 2;
7599                 tinfo->curr.protocol_version = 2;
7600                 tinfo->curr.transport_version = 2;
7601                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7602                                     targ, CAM_LUN_WILDCARD,
7603                                     'A', ROLE_INITIATOR);
7604                 tstate->tagenable &= ~target_mask;
7605                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7606                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7607                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7608                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7609                                  /*paused*/TRUE);
7610         }
7611         return (0);
7612 }
7613
7614 /*
7615  * Parse device configuration information.
7616  */
7617 int
7618 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
7619 {
7620         int targ;
7621         int max_targ;
7622
7623         max_targ = sc->max_targets & CFMAXTARG;
7624         ahd->our_id = sc->brtime_id & CFSCSIID;
7625
7626         /*
7627          * Allocate a tstate to house information for our
7628          * initiator presence on the bus as well as the user
7629          * data for any target mode initiator.
7630          */
7631         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7632                 printk("%s: unable to allocate ahd_tmode_tstate.  "
7633                        "Failing attach\n", ahd_name(ahd));
7634                 return (ENOMEM);
7635         }
7636
7637         for (targ = 0; targ < max_targ; targ++) {
7638                 struct   ahd_devinfo devinfo;
7639                 struct   ahd_initiator_tinfo *tinfo;
7640                 struct   ahd_transinfo *user_tinfo;
7641                 struct   ahd_tmode_tstate *tstate;
7642                 uint16_t target_mask;
7643
7644                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7645                                             targ, &tstate);
7646                 user_tinfo = &tinfo->user;
7647
7648                 /*
7649                  * We support SPC2 and SPI4.
7650                  */
7651                 tinfo->user.protocol_version = 4;
7652                 tinfo->user.transport_version = 4;
7653
7654                 target_mask = 0x01 << targ;
7655                 ahd->user_discenable &= ~target_mask;
7656                 tstate->discenable &= ~target_mask;
7657                 ahd->user_tagenable &= ~target_mask;
7658                 if (sc->device_flags[targ] & CFDISC) {
7659                         tstate->discenable |= target_mask;
7660                         ahd->user_discenable |= target_mask;
7661                         ahd->user_tagenable |= target_mask;
7662                 } else {
7663                         /*
7664                          * Cannot be packetized without disconnection.
7665                          */
7666                         sc->device_flags[targ] &= ~CFPACKETIZED;
7667                 }
7668
7669                 user_tinfo->ppr_options = 0;
7670                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
7671                 if (user_tinfo->period < CFXFER_ASYNC) {
7672                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
7673                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
7674                         user_tinfo->offset = MAX_OFFSET;
7675                 } else  {
7676                         user_tinfo->offset = 0;
7677                         user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
7678                 }
7679 #ifdef AHD_FORCE_160
7680                 if (user_tinfo->period <= AHD_SYNCRATE_160)
7681                         user_tinfo->period = AHD_SYNCRATE_DT;
7682 #endif
7683
7684                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
7685                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
7686                                                 |  MSG_EXT_PPR_WR_FLOW
7687                                                 |  MSG_EXT_PPR_HOLD_MCS
7688                                                 |  MSG_EXT_PPR_IU_REQ;
7689                         if ((ahd->features & AHD_RTI) != 0)
7690                                 user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
7691                 }
7692
7693                 if ((sc->device_flags[targ] & CFQAS) != 0)
7694                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
7695
7696                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
7697                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
7698                 else
7699                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
7700 #ifdef AHD_DEBUG
7701                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7702                         printk("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
7703                                user_tinfo->period, user_tinfo->offset,
7704                                user_tinfo->ppr_options);
7705 #endif
7706                 /*
7707                  * Start out Async/Narrow/Untagged and with
7708                  * conservative protocol support.
7709                  */
7710                 tstate->tagenable &= ~target_mask;
7711                 tinfo->goal.protocol_version = 2;
7712                 tinfo->goal.transport_version = 2;
7713                 tinfo->curr.protocol_version = 2;
7714                 tinfo->curr.transport_version = 2;
7715                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7716                                     targ, CAM_LUN_WILDCARD,
7717                                     'A', ROLE_INITIATOR);
7718                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7719                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7720                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7721                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7722                                  /*paused*/TRUE);
7723         }
7724
7725         ahd->flags &= ~AHD_SPCHK_ENB_A;
7726         if (sc->bios_control & CFSPARITY)
7727                 ahd->flags |= AHD_SPCHK_ENB_A;
7728
7729         ahd->flags &= ~AHD_RESET_BUS_A;
7730         if (sc->bios_control & CFRESETB)
7731                 ahd->flags |= AHD_RESET_BUS_A;
7732
7733         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
7734         if (sc->bios_control & CFEXTEND)
7735                 ahd->flags |= AHD_EXTENDED_TRANS_A;
7736
7737         ahd->flags &= ~AHD_BIOS_ENABLED;
7738         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
7739                 ahd->flags |= AHD_BIOS_ENABLED;
7740
7741         ahd->flags &= ~AHD_STPWLEVEL_A;
7742         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
7743                 ahd->flags |= AHD_STPWLEVEL_A;
7744
7745         return (0);
7746 }
7747
7748 /*
7749  * Parse device configuration information.
7750  */
7751 int
7752 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
7753 {
7754         int error;
7755
7756         error = ahd_verify_vpd_cksum(vpd);
7757         if (error == 0)
7758                 return (EINVAL);
7759         if ((vpd->bios_flags & VPDBOOTHOST) != 0)
7760                 ahd->flags |= AHD_BOOT_CHANNEL;
7761         return (0);
7762 }
7763
7764 void
7765 ahd_intr_enable(struct ahd_softc *ahd, int enable)
7766 {
7767         u_int hcntrl;
7768
7769         hcntrl = ahd_inb(ahd, HCNTRL);
7770         hcntrl &= ~INTEN;
7771         ahd->pause &= ~INTEN;
7772         ahd->unpause &= ~INTEN;
7773         if (enable) {
7774                 hcntrl |= INTEN;
7775                 ahd->pause |= INTEN;
7776                 ahd->unpause |= INTEN;
7777         }
7778         ahd_outb(ahd, HCNTRL, hcntrl);
7779 }
7780
7781 static void
7782 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7783                              u_int mincmds)
7784 {
7785         if (timer > AHD_TIMER_MAX_US)
7786                 timer = AHD_TIMER_MAX_US;
7787         ahd->int_coalescing_timer = timer;
7788
7789         if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7790                 maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7791         if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7792                 mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7793         ahd->int_coalescing_maxcmds = maxcmds;
7794         ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7795         ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7796         ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7797 }
7798
7799 static void
7800 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7801 {
7802
7803         ahd->hs_mailbox &= ~ENINT_COALESCE;
7804         if (enable)
7805                 ahd->hs_mailbox |= ENINT_COALESCE;
7806         ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7807         ahd_flush_device_writes(ahd);
7808         ahd_run_qoutfifo(ahd);
7809 }
7810
7811 /*
7812  * Ensure that the card is paused in a location
7813  * outside of all critical sections and that all
7814  * pending work is completed prior to returning.
7815  * This routine should only be called from outside
7816  * an interrupt context.
7817  */
7818 void
7819 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7820 {
7821         u_int intstat;
7822         u_int maxloops;
7823
7824         maxloops = 1000;
7825         ahd->flags |= AHD_ALL_INTERRUPTS;
7826         ahd_pause(ahd);
7827         /*
7828          * Freeze the outgoing selections.  We do this only
7829          * until we are safely paused without further selections
7830          * pending.
7831          */
7832         ahd->qfreeze_cnt--;
7833         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7834         ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7835         do {
7836
7837                 ahd_unpause(ahd);
7838                 /*
7839                  * Give the sequencer some time to service
7840                  * any active selections.
7841                  */
7842                 ahd_delay(500);
7843
7844                 ahd_intr(ahd);
7845                 ahd_pause(ahd);
7846                 intstat = ahd_inb(ahd, INTSTAT);
7847                 if ((intstat & INT_PEND) == 0) {
7848                         ahd_clear_critical_section(ahd);
7849                         intstat = ahd_inb(ahd, INTSTAT);
7850                 }
7851         } while (--maxloops
7852               && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7853               && ((intstat & INT_PEND) != 0
7854                || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7855                || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7856
7857         if (maxloops == 0) {
7858                 printk("Infinite interrupt loop, INTSTAT = %x",
7859                       ahd_inb(ahd, INTSTAT));
7860         }
7861         ahd->qfreeze_cnt++;
7862         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7863
7864         ahd_flush_qoutfifo(ahd);
7865
7866         ahd->flags &= ~AHD_ALL_INTERRUPTS;
7867 }
7868
7869 #ifdef CONFIG_PM
7870 int
7871 ahd_suspend(struct ahd_softc *ahd)
7872 {
7873
7874         ahd_pause_and_flushwork(ahd);
7875
7876         if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7877                 ahd_unpause(ahd);
7878                 return (EBUSY);
7879         }
7880         ahd_shutdown(ahd);
7881         return (0);
7882 }
7883
7884 void
7885 ahd_resume(struct ahd_softc *ahd)
7886 {
7887
7888         ahd_reset(ahd, /*reinit*/TRUE);
7889         ahd_intr_enable(ahd, TRUE);
7890         ahd_restart(ahd);
7891 }
7892 #endif
7893
7894 /************************** Busy Target Table *********************************/
7895 /*
7896  * Set SCBPTR to the SCB that contains the busy
7897  * table entry for TCL.  Return the offset into
7898  * the SCB that contains the entry for TCL.
7899  * saved_scbid is dereferenced and set to the
7900  * scbid that should be restored once manipualtion
7901  * of the TCL entry is complete.
7902  */
7903 static inline u_int
7904 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7905 {
7906         /*
7907          * Index to the SCB that contains the busy entry.
7908          */
7909         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7910         *saved_scbid = ahd_get_scbptr(ahd);
7911         ahd_set_scbptr(ahd, TCL_LUN(tcl)
7912                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7913
7914         /*
7915          * And now calculate the SCB offset to the entry.
7916          * Each entry is 2 bytes wide, hence the
7917          * multiplication by 2.
7918          */
7919         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
7920 }
7921
7922 /*
7923  * Return the untagged transaction id for a given target/channel lun.
7924  */
7925 static u_int
7926 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
7927 {
7928         u_int scbid;
7929         u_int scb_offset;
7930         u_int saved_scbptr;
7931
7932         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7933         scbid = ahd_inw_scbram(ahd, scb_offset);
7934         ahd_set_scbptr(ahd, saved_scbptr);
7935         return (scbid);
7936 }
7937
7938 static void
7939 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
7940 {
7941         u_int scb_offset;
7942         u_int saved_scbptr;
7943
7944         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7945         ahd_outw(ahd, scb_offset, scbid);
7946         ahd_set_scbptr(ahd, saved_scbptr);
7947 }
7948
7949 /************************** SCB and SCB queue management **********************/
7950 static int
7951 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
7952               char channel, int lun, u_int tag, role_t role)
7953 {
7954         int targ = SCB_GET_TARGET(ahd, scb);
7955         char chan = SCB_GET_CHANNEL(ahd, scb);
7956         int slun = SCB_GET_LUN(scb);
7957         int match;
7958
7959         match = ((chan == channel) || (channel == ALL_CHANNELS));
7960         if (match != 0)
7961                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
7962         if (match != 0)
7963                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
7964         if (match != 0) {
7965 #ifdef AHD_TARGET_MODE
7966                 int group;
7967
7968                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
7969                 if (role == ROLE_INITIATOR) {
7970                         match = (group != XPT_FC_GROUP_TMODE)
7971                               && ((tag == SCB_GET_TAG(scb))
7972                                || (tag == SCB_LIST_NULL));
7973                 } else if (role == ROLE_TARGET) {
7974                         match = (group == XPT_FC_GROUP_TMODE)
7975                               && ((tag == scb->io_ctx->csio.tag_id)
7976                                || (tag == SCB_LIST_NULL));
7977                 }
7978 #else /* !AHD_TARGET_MODE */
7979                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
7980 #endif /* AHD_TARGET_MODE */
7981         }
7982
7983         return match;
7984 }
7985
7986 static void
7987 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
7988 {
7989         int     target;
7990         char    channel;
7991         int     lun;
7992
7993         target = SCB_GET_TARGET(ahd, scb);
7994         lun = SCB_GET_LUN(scb);
7995         channel = SCB_GET_CHANNEL(ahd, scb);
7996
7997         ahd_search_qinfifo(ahd, target, channel, lun,
7998                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
7999                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
8000
8001         ahd_platform_freeze_devq(ahd, scb);
8002 }
8003
8004 void
8005 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
8006 {
8007         struct scb      *prev_scb;
8008         ahd_mode_state   saved_modes;
8009
8010         saved_modes = ahd_save_modes(ahd);
8011         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8012         prev_scb = NULL;
8013         if (ahd_qinfifo_count(ahd) != 0) {
8014                 u_int prev_tag;
8015                 u_int prev_pos;
8016
8017                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
8018                 prev_tag = ahd->qinfifo[prev_pos];
8019                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
8020         }
8021         ahd_qinfifo_requeue(ahd, prev_scb, scb);
8022         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8023         ahd_restore_modes(ahd, saved_modes);
8024 }
8025
8026 static void
8027 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
8028                     struct scb *scb)
8029 {
8030         if (prev_scb == NULL) {
8031                 uint32_t busaddr;
8032
8033                 busaddr = ahd_le32toh(scb->hscb->hscb_busaddr);
8034                 ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8035         } else {
8036                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
8037                 ahd_sync_scb(ahd, prev_scb,
8038                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8039         }
8040         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
8041         ahd->qinfifonext++;
8042         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
8043         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8044 }
8045
8046 static int
8047 ahd_qinfifo_count(struct ahd_softc *ahd)
8048 {
8049         u_int qinpos;
8050         u_int wrap_qinpos;
8051         u_int wrap_qinfifonext;
8052
8053         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
8054         qinpos = ahd_get_snscb_qoff(ahd);
8055         wrap_qinpos = AHD_QIN_WRAP(qinpos);
8056         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
8057         if (wrap_qinfifonext >= wrap_qinpos)
8058                 return (wrap_qinfifonext - wrap_qinpos);
8059         else
8060                 return (wrap_qinfifonext
8061                       + ARRAY_SIZE(ahd->qinfifo) - wrap_qinpos);
8062 }
8063
8064 static void
8065 ahd_reset_cmds_pending(struct ahd_softc *ahd)
8066 {
8067         struct          scb *scb;
8068         ahd_mode_state  saved_modes;
8069         u_int           pending_cmds;
8070
8071         saved_modes = ahd_save_modes(ahd);
8072         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8073
8074         /*
8075          * Don't count any commands as outstanding that the
8076          * sequencer has already marked for completion.
8077          */
8078         ahd_flush_qoutfifo(ahd);
8079
8080         pending_cmds = 0;
8081         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
8082                 pending_cmds++;
8083         }
8084         ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
8085         ahd_restore_modes(ahd, saved_modes);
8086         ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
8087 }
8088
8089 static void
8090 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
8091 {
8092         cam_status ostat;
8093         cam_status cstat;
8094
8095         ostat = ahd_get_transaction_status(scb);
8096         if (ostat == CAM_REQ_INPROG)
8097                 ahd_set_transaction_status(scb, status);
8098         cstat = ahd_get_transaction_status(scb);
8099         if (cstat != CAM_REQ_CMP)
8100                 ahd_freeze_scb(scb);
8101         ahd_done(ahd, scb);
8102 }
8103
8104 int
8105 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
8106                    int lun, u_int tag, role_t role, uint32_t status,
8107                    ahd_search_action action)
8108 {
8109         struct scb      *scb;
8110         struct scb      *mk_msg_scb;
8111         struct scb      *prev_scb;
8112         ahd_mode_state   saved_modes;
8113         u_int            qinstart;
8114         u_int            qinpos;
8115         u_int            qintail;
8116         u_int            tid_next;
8117         u_int            tid_prev;
8118         u_int            scbid;
8119         u_int            seq_flags2;
8120         u_int            savedscbptr;
8121         uint32_t         busaddr;
8122         int              found;
8123         int              targets;
8124
8125         /* Must be in CCHAN mode */
8126         saved_modes = ahd_save_modes(ahd);
8127         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8128
8129         /*
8130          * Halt any pending SCB DMA.  The sequencer will reinitiate
8131          * this dma if the qinfifo is not empty once we unpause.
8132          */
8133         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
8134          == (CCARREN|CCSCBEN|CCSCBDIR)) {
8135                 ahd_outb(ahd, CCSCBCTL,
8136                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
8137                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
8138                         ;
8139         }
8140         /* Determine sequencer's position in the qinfifo. */
8141         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
8142         qinstart = ahd_get_snscb_qoff(ahd);
8143         qinpos = AHD_QIN_WRAP(qinstart);
8144         found = 0;
8145         prev_scb = NULL;
8146
8147         if (action == SEARCH_PRINT) {
8148                 printk("qinstart = %d qinfifonext = %d\nQINFIFO:",
8149                        qinstart, ahd->qinfifonext);
8150         }
8151
8152         /*
8153          * Start with an empty queue.  Entries that are not chosen
8154          * for removal will be re-added to the queue as we go.
8155          */
8156         ahd->qinfifonext = qinstart;
8157         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
8158         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8159
8160         while (qinpos != qintail) {
8161                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
8162                 if (scb == NULL) {
8163                         printk("qinpos = %d, SCB index = %d\n",
8164                                 qinpos, ahd->qinfifo[qinpos]);
8165                         panic("Loop 1\n");
8166                 }
8167
8168                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
8169                         /*
8170                          * We found an scb that needs to be acted on.
8171                          */
8172                         found++;
8173                         switch (action) {
8174                         case SEARCH_COMPLETE:
8175                                 if ((scb->flags & SCB_ACTIVE) == 0)
8176                                         printk("Inactive SCB in qinfifo\n");
8177                                 ahd_done_with_status(ahd, scb, status);
8178                                 fallthrough;
8179                         case SEARCH_REMOVE:
8180                                 break;
8181                         case SEARCH_PRINT:
8182                                 printk(" 0x%x", ahd->qinfifo[qinpos]);
8183                                 fallthrough;
8184                         case SEARCH_COUNT:
8185                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
8186                                 prev_scb = scb;
8187                                 break;
8188                         }
8189                 } else {
8190                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
8191                         prev_scb = scb;
8192                 }
8193                 qinpos = AHD_QIN_WRAP(qinpos+1);
8194         }
8195
8196         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8197
8198         if (action == SEARCH_PRINT)
8199                 printk("\nWAITING_TID_QUEUES:\n");
8200
8201         /*
8202          * Search waiting for selection lists.  We traverse the
8203          * list of "their ids" waiting for selection and, if
8204          * appropriate, traverse the SCBs of each "their id"
8205          * looking for matches.
8206          */
8207         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8208         seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
8209         if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
8210                 scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
8211                 mk_msg_scb = ahd_lookup_scb(ahd, scbid);
8212         } else
8213                 mk_msg_scb = NULL;
8214         savedscbptr = ahd_get_scbptr(ahd);
8215         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
8216         tid_prev = SCB_LIST_NULL;
8217         targets = 0;
8218         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
8219                 u_int tid_head;
8220                 u_int tid_tail;
8221
8222                 targets++;
8223                 if (targets > AHD_NUM_TARGETS)
8224                         panic("TID LIST LOOP");
8225
8226                 if (scbid >= ahd->scb_data.numscbs) {
8227                         printk("%s: Waiting TID List inconsistency. "
8228                                "SCB index == 0x%x, yet numscbs == 0x%x.",
8229                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8230                         ahd_dump_card_state(ahd);
8231                         panic("for safety");
8232                 }
8233                 scb = ahd_lookup_scb(ahd, scbid);
8234                 if (scb == NULL) {
8235                         printk("%s: SCB = 0x%x Not Active!\n",
8236                                ahd_name(ahd), scbid);
8237                         panic("Waiting TID List traversal\n");
8238                 }
8239                 ahd_set_scbptr(ahd, scbid);
8240                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
8241                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8242                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
8243                         tid_prev = scbid;
8244                         continue;
8245                 }
8246
8247                 /*
8248                  * We found a list of scbs that needs to be searched.
8249                  */
8250                 if (action == SEARCH_PRINT)
8251                         printk("       %d ( ", SCB_GET_TARGET(ahd, scb));
8252                 tid_head = scbid;
8253                 found += ahd_search_scb_list(ahd, target, channel,
8254                                              lun, tag, role, status,
8255                                              action, &tid_head, &tid_tail,
8256                                              SCB_GET_TARGET(ahd, scb));
8257                 /*
8258                  * Check any MK_MESSAGE SCB that is still waiting to
8259                  * enter this target's waiting for selection queue.
8260                  */
8261                 if (mk_msg_scb != NULL
8262                  && ahd_match_scb(ahd, mk_msg_scb, target, channel,
8263                                   lun, tag, role)) {
8264
8265                         /*
8266                          * We found an scb that needs to be acted on.
8267                          */
8268                         found++;
8269                         switch (action) {
8270                         case SEARCH_COMPLETE:
8271                                 if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
8272                                         printk("Inactive SCB pending MK_MSG\n");
8273                                 ahd_done_with_status(ahd, mk_msg_scb, status);
8274                                 fallthrough;
8275                         case SEARCH_REMOVE:
8276                         {
8277                                 u_int tail_offset;
8278
8279                                 printk("Removing MK_MSG scb\n");
8280
8281                                 /*
8282                                  * Reset our tail to the tail of the
8283                                  * main per-target list.
8284                                  */
8285                                 tail_offset = WAITING_SCB_TAILS
8286                                     + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
8287                                 ahd_outw(ahd, tail_offset, tid_tail);
8288
8289                                 seq_flags2 &= ~PENDING_MK_MESSAGE;
8290                                 ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8291                                 ahd_outw(ahd, CMDS_PENDING,
8292                                          ahd_inw(ahd, CMDS_PENDING)-1);
8293                                 mk_msg_scb = NULL;
8294                                 break;
8295                         }
8296                         case SEARCH_PRINT:
8297                                 printk(" 0x%x", SCB_GET_TAG(scb));
8298                                 fallthrough;
8299                         case SEARCH_COUNT:
8300                                 break;
8301                         }
8302                 }
8303
8304                 if (mk_msg_scb != NULL
8305                  && SCBID_IS_NULL(tid_head)
8306                  && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8307                                   SCB_LIST_NULL, ROLE_UNKNOWN)) {
8308
8309                         /*
8310                          * When removing the last SCB for a target
8311                          * queue with a pending MK_MESSAGE scb, we
8312                          * must queue the MK_MESSAGE scb.
8313                          */
8314                         printk("Queueing mk_msg_scb\n");
8315                         tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
8316                         seq_flags2 &= ~PENDING_MK_MESSAGE;
8317                         ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8318                         mk_msg_scb = NULL;
8319                 }
8320                 if (tid_head != scbid)
8321                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
8322                 if (!SCBID_IS_NULL(tid_head))
8323                         tid_prev = tid_head;
8324                 if (action == SEARCH_PRINT)
8325                         printk(")\n");
8326         }
8327
8328         /* Restore saved state. */
8329         ahd_set_scbptr(ahd, savedscbptr);
8330         ahd_restore_modes(ahd, saved_modes);
8331         return (found);
8332 }
8333
8334 static int
8335 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
8336                     int lun, u_int tag, role_t role, uint32_t status,
8337                     ahd_search_action action, u_int *list_head,
8338                     u_int *list_tail, u_int tid)
8339 {
8340         struct  scb *scb;
8341         u_int   scbid;
8342         u_int   next;
8343         u_int   prev;
8344         int     found;
8345
8346         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8347         found = 0;
8348         prev = SCB_LIST_NULL;
8349         next = *list_head;
8350         *list_tail = SCB_LIST_NULL;
8351         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
8352                 if (scbid >= ahd->scb_data.numscbs) {
8353                         printk("%s:SCB List inconsistency. "
8354                                "SCB == 0x%x, yet numscbs == 0x%x.",
8355                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8356                         ahd_dump_card_state(ahd);
8357                         panic("for safety");
8358                 }
8359                 scb = ahd_lookup_scb(ahd, scbid);
8360                 if (scb == NULL) {
8361                         printk("%s: SCB = %d Not Active!\n",
8362                                ahd_name(ahd), scbid);
8363                         panic("Waiting List traversal\n");
8364                 }
8365                 ahd_set_scbptr(ahd, scbid);
8366                 *list_tail = scbid;
8367                 next = ahd_inw_scbram(ahd, SCB_NEXT);
8368                 if (ahd_match_scb(ahd, scb, target, channel,
8369                                   lun, SCB_LIST_NULL, role) == 0) {
8370                         prev = scbid;
8371                         continue;
8372                 }
8373                 found++;
8374                 switch (action) {
8375                 case SEARCH_COMPLETE:
8376                         if ((scb->flags & SCB_ACTIVE) == 0)
8377                                 printk("Inactive SCB in Waiting List\n");
8378                         ahd_done_with_status(ahd, scb, status);
8379                         fallthrough;
8380                 case SEARCH_REMOVE:
8381                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
8382                         *list_tail = prev;
8383                         if (SCBID_IS_NULL(prev))
8384                                 *list_head = next;
8385                         break;
8386                 case SEARCH_PRINT:
8387                         printk("0x%x ", scbid);
8388                         fallthrough;
8389                 case SEARCH_COUNT:
8390                         prev = scbid;
8391                         break;
8392                 }
8393                 if (found > AHD_SCB_MAX)
8394                         panic("SCB LIST LOOP");
8395         }
8396         if (action == SEARCH_COMPLETE
8397          || action == SEARCH_REMOVE)
8398                 ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
8399         return (found);
8400 }
8401
8402 static void
8403 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
8404                     u_int tid_cur, u_int tid_next)
8405 {
8406         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8407
8408         if (SCBID_IS_NULL(tid_cur)) {
8409
8410                 /* Bypass current TID list */
8411                 if (SCBID_IS_NULL(tid_prev)) {
8412                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
8413                 } else {
8414                         ahd_set_scbptr(ahd, tid_prev);
8415                         ahd_outw(ahd, SCB_NEXT2, tid_next);
8416                 }
8417                 if (SCBID_IS_NULL(tid_next))
8418                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
8419         } else {
8420
8421                 /* Stitch through tid_cur */
8422                 if (SCBID_IS_NULL(tid_prev)) {
8423                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
8424                 } else {
8425                         ahd_set_scbptr(ahd, tid_prev);
8426                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
8427                 }
8428                 ahd_set_scbptr(ahd, tid_cur);
8429                 ahd_outw(ahd, SCB_NEXT2, tid_next);
8430
8431                 if (SCBID_IS_NULL(tid_next))
8432                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
8433         }
8434 }
8435
8436 /*
8437  * Manipulate the waiting for selection list and return the
8438  * scb that follows the one that we remove.
8439  */
8440 static u_int
8441 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
8442              u_int prev, u_int next, u_int tid)
8443 {
8444         u_int tail_offset;
8445
8446         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8447         if (!SCBID_IS_NULL(prev)) {
8448                 ahd_set_scbptr(ahd, prev);
8449                 ahd_outw(ahd, SCB_NEXT, next);
8450         }
8451
8452         /*
8453          * SCBs that have MK_MESSAGE set in them may
8454          * cause the tail pointer to be updated without
8455          * setting the next pointer of the previous tail.
8456          * Only clear the tail if the removed SCB was
8457          * the tail.
8458          */
8459         tail_offset = WAITING_SCB_TAILS + (2 * tid);
8460         if (SCBID_IS_NULL(next)
8461          && ahd_inw(ahd, tail_offset) == scbid)
8462                 ahd_outw(ahd, tail_offset, prev);
8463
8464         ahd_add_scb_to_free_list(ahd, scbid);
8465         return (next);
8466 }
8467
8468 /*
8469  * Add the SCB as selected by SCBPTR onto the on chip list of
8470  * free hardware SCBs.  This list is empty/unused if we are not
8471  * performing SCB paging.
8472  */
8473 static void
8474 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
8475 {
8476 /* XXX Need some other mechanism to designate "free". */
8477         /*
8478          * Invalidate the tag so that our abort
8479          * routines don't think it's active.
8480         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
8481          */
8482 }
8483
8484 /******************************** Error Handling ******************************/
8485 /*
8486  * Abort all SCBs that match the given description (target/channel/lun/tag),
8487  * setting their status to the passed in status if the status has not already
8488  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
8489  * is paused before it is called.
8490  */
8491 static int
8492 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
8493                int lun, u_int tag, role_t role, uint32_t status)
8494 {
8495         struct          scb *scbp;
8496         struct          scb *scbp_next;
8497         u_int           i, j;
8498         u_int           maxtarget;
8499         u_int           minlun;
8500         u_int           maxlun;
8501         int             found;
8502         ahd_mode_state  saved_modes;
8503
8504         /* restore this when we're done */
8505         saved_modes = ahd_save_modes(ahd);
8506         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8507
8508         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
8509                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
8510
8511         /*
8512          * Clean out the busy target table for any untagged commands.
8513          */
8514         i = 0;
8515         maxtarget = 16;
8516         if (target != CAM_TARGET_WILDCARD) {
8517                 i = target;
8518                 if (channel == 'B')
8519                         i += 8;
8520                 maxtarget = i + 1;
8521         }
8522
8523         if (lun == CAM_LUN_WILDCARD) {
8524                 minlun = 0;
8525                 maxlun = AHD_NUM_LUNS_NONPKT;
8526         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
8527                 minlun = maxlun = 0;
8528         } else {
8529                 minlun = lun;
8530                 maxlun = lun + 1;
8531         }
8532
8533         if (role != ROLE_TARGET) {
8534                 for (;i < maxtarget; i++) {
8535                         for (j = minlun;j < maxlun; j++) {
8536                                 u_int scbid;
8537                                 u_int tcl;
8538
8539                                 tcl = BUILD_TCL_RAW(i, 'A', j);
8540                                 scbid = ahd_find_busy_tcl(ahd, tcl);
8541                                 scbp = ahd_lookup_scb(ahd, scbid);
8542                                 if (scbp == NULL
8543                                  || ahd_match_scb(ahd, scbp, target, channel,
8544                                                   lun, tag, role) == 0)
8545                                         continue;
8546                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
8547                         }
8548                 }
8549         }
8550
8551         /*
8552          * Don't abort commands that have already completed,
8553          * but haven't quite made it up to the host yet.
8554          */
8555         ahd_flush_qoutfifo(ahd);
8556
8557         /*
8558          * Go through the pending CCB list and look for
8559          * commands for this target that are still active.
8560          * These are other tagged commands that were
8561          * disconnected when the reset occurred.
8562          */
8563         scbp_next = LIST_FIRST(&ahd->pending_scbs);
8564         while (scbp_next != NULL) {
8565                 scbp = scbp_next;
8566                 scbp_next = LIST_NEXT(scbp, pending_links);
8567                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
8568                         cam_status ostat;
8569
8570                         ostat = ahd_get_transaction_status(scbp);
8571                         if (ostat == CAM_REQ_INPROG)
8572                                 ahd_set_transaction_status(scbp, status);
8573                         if (ahd_get_transaction_status(scbp) != CAM_REQ_CMP)
8574                                 ahd_freeze_scb(scbp);
8575                         if ((scbp->flags & SCB_ACTIVE) == 0)
8576                                 printk("Inactive SCB on pending list\n");
8577                         ahd_done(ahd, scbp);
8578                         found++;
8579                 }
8580         }
8581         ahd_restore_modes(ahd, saved_modes);
8582         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
8583         ahd->flags |= AHD_UPDATE_PEND_CMDS;
8584         return found;
8585 }
8586
8587 static void
8588 ahd_reset_current_bus(struct ahd_softc *ahd)
8589 {
8590         uint8_t scsiseq;
8591
8592         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8593         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
8594         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
8595         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
8596         ahd_flush_device_writes(ahd);
8597         ahd_delay(AHD_BUSRESET_DELAY);
8598         /* Turn off the bus reset */
8599         ahd_outb(ahd, SCSISEQ0, scsiseq);
8600         ahd_flush_device_writes(ahd);
8601         ahd_delay(AHD_BUSRESET_DELAY);
8602         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
8603                 /*
8604                  * 2A Razor #474
8605                  * Certain chip state is not cleared for
8606                  * SCSI bus resets that we initiate, so
8607                  * we must reset the chip.
8608                  */
8609                 ahd_reset(ahd, /*reinit*/TRUE);
8610                 ahd_intr_enable(ahd, /*enable*/TRUE);
8611                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8612         }
8613
8614         ahd_clear_intstat(ahd);
8615 }
8616
8617 int
8618 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
8619 {
8620         struct  ahd_devinfo caminfo;
8621         u_int   initiator;
8622         u_int   target;
8623         u_int   max_scsiid;
8624         int     found;
8625         u_int   fifo;
8626         u_int   next_fifo;
8627         uint8_t scsiseq;
8628
8629         /*
8630          * Check if the last bus reset is cleared
8631          */
8632         if (ahd->flags & AHD_BUS_RESET_ACTIVE) {
8633                 printk("%s: bus reset still active\n",
8634                        ahd_name(ahd));
8635                 return 0;
8636         }
8637         ahd->flags |= AHD_BUS_RESET_ACTIVE;
8638
8639         ahd->pending_device = NULL;
8640
8641         ahd_compile_devinfo(&caminfo,
8642                             CAM_TARGET_WILDCARD,
8643                             CAM_TARGET_WILDCARD,
8644                             CAM_LUN_WILDCARD,
8645                             channel, ROLE_UNKNOWN);
8646         ahd_pause(ahd);
8647
8648         /* Make sure the sequencer is in a safe location. */
8649         ahd_clear_critical_section(ahd);
8650
8651         /*
8652          * Run our command complete fifos to ensure that we perform
8653          * completion processing on any commands that 'completed'
8654          * before the reset occurred.
8655          */
8656         ahd_run_qoutfifo(ahd);
8657 #ifdef AHD_TARGET_MODE
8658         if ((ahd->flags & AHD_TARGETROLE) != 0) {
8659                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
8660         }
8661 #endif
8662         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8663
8664         /*
8665          * Disable selections so no automatic hardware
8666          * functions will modify chip state.
8667          */
8668         ahd_outb(ahd, SCSISEQ0, 0);
8669         ahd_outb(ahd, SCSISEQ1, 0);
8670
8671         /*
8672          * Safely shut down our DMA engines.  Always start with
8673          * the FIFO that is not currently active (if any are
8674          * actively connected).
8675          */
8676         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
8677         if (next_fifo > CURRFIFO_1)
8678                 /* If disconneced, arbitrarily start with FIFO1. */
8679                 next_fifo = fifo = 0;
8680         do {
8681                 next_fifo ^= CURRFIFO_1;
8682                 ahd_set_modes(ahd, next_fifo, next_fifo);
8683                 ahd_outb(ahd, DFCNTRL,
8684                          ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
8685                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
8686                         ahd_delay(10);
8687                 /*
8688                  * Set CURRFIFO to the now inactive channel.
8689                  */
8690                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8691                 ahd_outb(ahd, DFFSTAT, next_fifo);
8692         } while (next_fifo != fifo);
8693
8694         /*
8695          * Reset the bus if we are initiating this reset
8696          */
8697         ahd_clear_msg_state(ahd);
8698         ahd_outb(ahd, SIMODE1,
8699                  ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
8700
8701         if (initiate_reset)
8702                 ahd_reset_current_bus(ahd);
8703
8704         ahd_clear_intstat(ahd);
8705
8706         /*
8707          * Clean up all the state information for the
8708          * pending transactions on this bus.
8709          */
8710         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
8711                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
8712                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
8713
8714         /*
8715          * Cleanup anything left in the FIFOs.
8716          */
8717         ahd_clear_fifo(ahd, 0);
8718         ahd_clear_fifo(ahd, 1);
8719
8720         /*
8721          * Clear SCSI interrupt status
8722          */
8723         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8724
8725         /*
8726          * Reenable selections
8727          */
8728         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8729         scsiseq = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8730         ahd_outb(ahd, SCSISEQ1, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
8731
8732         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8733 #ifdef AHD_TARGET_MODE
8734         /*
8735          * Send an immediate notify ccb to all target more peripheral
8736          * drivers affected by this action.
8737          */
8738         for (target = 0; target <= max_scsiid; target++) {
8739                 struct ahd_tmode_tstate* tstate;
8740                 u_int lun;
8741
8742                 tstate = ahd->enabled_targets[target];
8743                 if (tstate == NULL)
8744                         continue;
8745                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
8746                         struct ahd_tmode_lstate* lstate;
8747
8748                         lstate = tstate->enabled_luns[lun];
8749                         if (lstate == NULL)
8750                                 continue;
8751
8752                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
8753                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
8754                         ahd_send_lstate_events(ahd, lstate);
8755                 }
8756         }
8757 #endif
8758         /*
8759          * Revert to async/narrow transfers until we renegotiate.
8760          */
8761         for (target = 0; target <= max_scsiid; target++) {
8762
8763                 if (ahd->enabled_targets[target] == NULL)
8764                         continue;
8765                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
8766                         struct ahd_devinfo devinfo;
8767
8768                         ahd_compile_devinfo(&devinfo, target, initiator,
8769                                             CAM_LUN_WILDCARD,
8770                                             'A', ROLE_UNKNOWN);
8771                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
8772                                       AHD_TRANS_CUR, /*paused*/TRUE);
8773                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
8774                                          /*offset*/0, /*ppr_options*/0,
8775                                          AHD_TRANS_CUR, /*paused*/TRUE);
8776                 }
8777         }
8778
8779         /* Notify the XPT that a bus reset occurred */
8780         ahd_send_async(ahd, caminfo.channel, CAM_TARGET_WILDCARD,
8781                        CAM_LUN_WILDCARD, AC_BUS_RESET);
8782
8783         ahd_restart(ahd);
8784
8785         return (found);
8786 }
8787
8788 /**************************** Statistics Processing ***************************/
8789 static void
8790 ahd_stat_timer(struct timer_list *t)
8791 {
8792         struct  ahd_softc *ahd = from_timer(ahd, t, stat_timer);
8793         u_long  s;
8794         int     enint_coal;
8795
8796         ahd_lock(ahd, &s);
8797
8798         enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8799         if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8800                 enint_coal |= ENINT_COALESCE;
8801         else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8802                 enint_coal &= ~ENINT_COALESCE;
8803
8804         if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8805                 ahd_enable_coalescing(ahd, enint_coal);
8806 #ifdef AHD_DEBUG
8807                 if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8808                         printk("%s: Interrupt coalescing "
8809                                "now %sabled. Cmds %d\n",
8810                                ahd_name(ahd),
8811                                (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8812                                ahd->cmdcmplt_total);
8813 #endif
8814         }
8815
8816         ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8817         ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8818         ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8819         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
8820         ahd_unlock(ahd, &s);
8821 }
8822
8823 /****************************** Status Processing *****************************/
8824
8825 static void
8826 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8827 {
8828         struct  hardware_scb *hscb;
8829         int     paused;
8830
8831         /*
8832          * The sequencer freezes its select-out queue
8833          * anytime a SCSI status error occurs.  We must
8834          * handle the error and increment our qfreeze count
8835          * to allow the sequencer to continue.  We don't
8836          * bother clearing critical sections here since all
8837          * operations are on data structures that the sequencer
8838          * is not touching once the queue is frozen.
8839          */
8840         hscb = scb->hscb;
8841
8842         if (ahd_is_paused(ahd)) {
8843                 paused = 1;
8844         } else {
8845                 paused = 0;
8846                 ahd_pause(ahd);
8847         }
8848
8849         /* Freeze the queue until the client sees the error. */
8850         ahd_freeze_devq(ahd, scb);
8851         ahd_freeze_scb(scb);
8852         ahd->qfreeze_cnt++;
8853         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8854
8855         if (paused == 0)
8856                 ahd_unpause(ahd);
8857
8858         /* Don't want to clobber the original sense code */
8859         if ((scb->flags & SCB_SENSE) != 0) {
8860                 /*
8861                  * Clear the SCB_SENSE Flag and perform
8862                  * a normal command completion.
8863                  */
8864                 scb->flags &= ~SCB_SENSE;
8865                 ahd_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8866                 ahd_done(ahd, scb);
8867                 return;
8868         }
8869         ahd_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8870         ahd_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8871         switch (hscb->shared_data.istatus.scsi_status) {
8872         case STATUS_PKT_SENSE:
8873         {
8874                 struct scsi_status_iu_header *siu;
8875
8876                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8877                 siu = (struct scsi_status_iu_header *)scb->sense_data;
8878                 ahd_set_scsi_status(scb, siu->status);
8879 #ifdef AHD_DEBUG
8880                 if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8881                         ahd_print_path(ahd, scb);
8882                         printk("SCB 0x%x Received PKT Status of 0x%x\n",
8883                                SCB_GET_TAG(scb), siu->status);
8884                         printk("\tflags = 0x%x, sense len = 0x%x, "
8885                                "pktfail = 0x%x\n",
8886                                siu->flags, scsi_4btoul(siu->sense_length),
8887                                scsi_4btoul(siu->pkt_failures_length));
8888                 }
8889 #endif
8890                 if ((siu->flags & SIU_RSPVALID) != 0) {
8891                         ahd_print_path(ahd, scb);
8892                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8893                                 printk("Unable to parse pkt_failures\n");
8894                         } else {
8895
8896                                 switch (SIU_PKTFAIL_CODE(siu)) {
8897                                 case SIU_PFC_NONE:
8898                                         printk("No packet failure found\n");
8899                                         break;
8900                                 case SIU_PFC_CIU_FIELDS_INVALID:
8901                                         printk("Invalid Command IU Field\n");
8902                                         break;
8903                                 case SIU_PFC_TMF_NOT_SUPPORTED:
8904                                         printk("TMF not supported\n");
8905                                         break;
8906                                 case SIU_PFC_TMF_FAILED:
8907                                         printk("TMF failed\n");
8908                                         break;
8909                                 case SIU_PFC_INVALID_TYPE_CODE:
8910                                         printk("Invalid L_Q Type code\n");
8911                                         break;
8912                                 case SIU_PFC_ILLEGAL_REQUEST:
8913                                         printk("Illegal request\n");
8914                                 default:
8915                                         break;
8916                                 }
8917                         }
8918                         if (siu->status == SCSI_STATUS_OK)
8919                                 ahd_set_transaction_status(scb,
8920                                                            CAM_REQ_CMP_ERR);
8921                 }
8922                 if ((siu->flags & SIU_SNSVALID) != 0) {
8923                         scb->flags |= SCB_PKT_SENSE;
8924 #ifdef AHD_DEBUG
8925                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
8926                                 printk("Sense data available\n");
8927 #endif
8928                 }
8929                 ahd_done(ahd, scb);
8930                 break;
8931         }
8932         case SCSI_STATUS_CMD_TERMINATED:
8933         case SCSI_STATUS_CHECK_COND:
8934         {
8935                 struct ahd_devinfo devinfo;
8936                 struct ahd_dma_seg *sg;
8937                 struct scsi_sense *sc;
8938                 struct ahd_initiator_tinfo *targ_info;
8939                 struct ahd_tmode_tstate *tstate;
8940                 struct ahd_transinfo *tinfo;
8941 #ifdef AHD_DEBUG
8942                 if (ahd_debug & AHD_SHOW_SENSE) {
8943                         ahd_print_path(ahd, scb);
8944                         printk("SCB %d: requests Check Status\n",
8945                                SCB_GET_TAG(scb));
8946                 }
8947 #endif
8948
8949                 if (ahd_perform_autosense(scb) == 0)
8950                         break;
8951
8952                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
8953                                     SCB_GET_TARGET(ahd, scb),
8954                                     SCB_GET_LUN(scb),
8955                                     SCB_GET_CHANNEL(ahd, scb),
8956                                     ROLE_INITIATOR);
8957                 targ_info = ahd_fetch_transinfo(ahd,
8958                                                 devinfo.channel,
8959                                                 devinfo.our_scsiid,
8960                                                 devinfo.target,
8961                                                 &tstate);
8962                 tinfo = &targ_info->curr;
8963                 sg = scb->sg_list;
8964                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
8965                 /*
8966                  * Save off the residual if there is one.
8967                  */
8968                 ahd_update_residual(ahd, scb);
8969 #ifdef AHD_DEBUG
8970                 if (ahd_debug & AHD_SHOW_SENSE) {
8971                         ahd_print_path(ahd, scb);
8972                         printk("Sending Sense\n");
8973                 }
8974 #endif
8975                 scb->sg_count = 0;
8976                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
8977                                   ahd_get_sense_bufsize(ahd, scb),
8978                                   /*last*/TRUE);
8979                 sc->opcode = REQUEST_SENSE;
8980                 sc->byte2 = 0;
8981                 if (tinfo->protocol_version <= SCSI_REV_2
8982                  && SCB_GET_LUN(scb) < 8)
8983                         sc->byte2 = SCB_GET_LUN(scb) << 5;
8984                 sc->unused[0] = 0;
8985                 sc->unused[1] = 0;
8986                 sc->length = ahd_get_sense_bufsize(ahd, scb);
8987                 sc->control = 0;
8988
8989                 /*
8990                  * We can't allow the target to disconnect.
8991                  * This will be an untagged transaction and
8992                  * having the target disconnect will make this
8993                  * transaction indestinguishable from outstanding
8994                  * tagged transactions.
8995                  */
8996                 hscb->control = 0;
8997
8998                 /*
8999                  * This request sense could be because the
9000                  * the device lost power or in some other
9001                  * way has lost our transfer negotiations.
9002                  * Renegotiate if appropriate.  Unit attention
9003                  * errors will be reported before any data
9004                  * phases occur.
9005                  */
9006                 if (ahd_get_residual(scb) == ahd_get_transfer_length(scb)) {
9007                         ahd_update_neg_request(ahd, &devinfo,
9008                                                tstate, targ_info,
9009                                                AHD_NEG_IF_NON_ASYNC);
9010                 }
9011                 if (tstate->auto_negotiate & devinfo.target_mask) {
9012                         hscb->control |= MK_MESSAGE;
9013                         scb->flags &=
9014                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
9015                         scb->flags |= SCB_AUTO_NEGOTIATE;
9016                 }
9017                 hscb->cdb_len = sizeof(*sc);
9018                 ahd_setup_data_scb(ahd, scb);
9019                 scb->flags |= SCB_SENSE;
9020                 ahd_queue_scb(ahd, scb);
9021                 break;
9022         }
9023         case SCSI_STATUS_OK:
9024                 printk("%s: Interrupted for status of 0???\n",
9025                        ahd_name(ahd));
9026                 fallthrough;
9027         default:
9028                 ahd_done(ahd, scb);
9029                 break;
9030         }
9031 }
9032
9033 static void
9034 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
9035 {
9036         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
9037                 ahd_handle_scsi_status(ahd, scb);
9038         } else {
9039                 ahd_calc_residual(ahd, scb);
9040                 ahd_done(ahd, scb);
9041         }
9042 }
9043
9044 /*
9045  * Calculate the residual for a just completed SCB.
9046  */
9047 static void
9048 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
9049 {
9050         struct hardware_scb *hscb;
9051         struct initiator_status *spkt;
9052         uint32_t sgptr;
9053         uint32_t resid_sgptr;
9054         uint32_t resid;
9055
9056         /*
9057          * 5 cases.
9058          * 1) No residual.
9059          *    SG_STATUS_VALID clear in sgptr.
9060          * 2) Transferless command
9061          * 3) Never performed any transfers.
9062          *    sgptr has SG_FULL_RESID set.
9063          * 4) No residual but target did not
9064          *    save data pointers after the
9065          *    last transfer, so sgptr was
9066          *    never updated.
9067          * 5) We have a partial residual.
9068          *    Use residual_sgptr to determine
9069          *    where we are.
9070          */
9071
9072         hscb = scb->hscb;
9073         sgptr = ahd_le32toh(hscb->sgptr);
9074         if ((sgptr & SG_STATUS_VALID) == 0)
9075                 /* Case 1 */
9076                 return;
9077         sgptr &= ~SG_STATUS_VALID;
9078
9079         if ((sgptr & SG_LIST_NULL) != 0)
9080                 /* Case 2 */
9081                 return;
9082
9083         /*
9084          * Residual fields are the same in both
9085          * target and initiator status packets,
9086          * so we can always use the initiator fields
9087          * regardless of the role for this SCB.
9088          */
9089         spkt = &hscb->shared_data.istatus;
9090         resid_sgptr = ahd_le32toh(spkt->residual_sgptr);
9091         if ((sgptr & SG_FULL_RESID) != 0) {
9092                 /* Case 3 */
9093                 resid = ahd_get_transfer_length(scb);
9094         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
9095                 /* Case 4 */
9096                 return;
9097         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
9098                 ahd_print_path(ahd, scb);
9099                 printk("data overrun detected Tag == 0x%x.\n",
9100                        SCB_GET_TAG(scb));
9101                 ahd_freeze_devq(ahd, scb);
9102                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
9103                 ahd_freeze_scb(scb);
9104                 return;
9105         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
9106                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
9107                 /* NOTREACHED */
9108         } else {
9109                 struct ahd_dma_seg *sg;
9110
9111                 /*
9112                  * Remainder of the SG where the transfer
9113                  * stopped.
9114                  */
9115                 resid = ahd_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
9116                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
9117
9118                 /* The residual sg_ptr always points to the next sg */
9119                 sg--;
9120
9121                 /*
9122                  * Add up the contents of all residual
9123                  * SG segments that are after the SG where
9124                  * the transfer stopped.
9125                  */
9126                 while ((ahd_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
9127                         sg++;
9128                         resid += ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
9129                 }
9130         }
9131         if ((scb->flags & SCB_SENSE) == 0)
9132                 ahd_set_residual(scb, resid);
9133         else
9134                 ahd_set_sense_residual(scb, resid);
9135
9136 #ifdef AHD_DEBUG
9137         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
9138                 ahd_print_path(ahd, scb);
9139                 printk("Handled %sResidual of %d bytes\n",
9140                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
9141         }
9142 #endif
9143 }
9144
9145 /******************************* Target Mode **********************************/
9146 #ifdef AHD_TARGET_MODE
9147 /*
9148  * Add a target mode event to this lun's queue
9149  */
9150 static void
9151 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
9152                        u_int initiator_id, u_int event_type, u_int event_arg)
9153 {
9154         struct ahd_tmode_event *event;
9155         int pending;
9156
9157         xpt_freeze_devq(lstate->path, /*count*/1);
9158         if (lstate->event_w_idx >= lstate->event_r_idx)
9159                 pending = lstate->event_w_idx - lstate->event_r_idx;
9160         else
9161                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
9162                         - (lstate->event_r_idx - lstate->event_w_idx);
9163
9164         if (event_type == EVENT_TYPE_BUS_RESET
9165          || event_type == MSG_BUS_DEV_RESET) {
9166                 /*
9167                  * Any earlier events are irrelevant, so reset our buffer.
9168                  * This has the effect of allowing us to deal with reset
9169                  * floods (an external device holding down the reset line)
9170                  * without losing the event that is really interesting.
9171                  */
9172                 lstate->event_r_idx = 0;
9173                 lstate->event_w_idx = 0;
9174                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
9175         }
9176
9177         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
9178                 xpt_print_path(lstate->path);
9179                 printk("immediate event %x:%x lost\n",
9180                        lstate->event_buffer[lstate->event_r_idx].event_type,
9181                        lstate->event_buffer[lstate->event_r_idx].event_arg);
9182                 lstate->event_r_idx++;
9183                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9184                         lstate->event_r_idx = 0;
9185                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
9186         }
9187
9188         event = &lstate->event_buffer[lstate->event_w_idx];
9189         event->initiator_id = initiator_id;
9190         event->event_type = event_type;
9191         event->event_arg = event_arg;
9192         lstate->event_w_idx++;
9193         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9194                 lstate->event_w_idx = 0;
9195 }
9196
9197 /*
9198  * Send any target mode events queued up waiting
9199  * for immediate notify resources.
9200  */
9201 void
9202 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
9203 {
9204         struct ccb_hdr *ccbh;
9205         struct ccb_immed_notify *inot;
9206
9207         while (lstate->event_r_idx != lstate->event_w_idx
9208             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
9209                 struct ahd_tmode_event *event;
9210
9211                 event = &lstate->event_buffer[lstate->event_r_idx];
9212                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
9213                 inot = (struct ccb_immed_notify *)ccbh;
9214                 switch (event->event_type) {
9215                 case EVENT_TYPE_BUS_RESET:
9216                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
9217                         break;
9218                 default:
9219                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
9220                         inot->message_args[0] = event->event_type;
9221                         inot->message_args[1] = event->event_arg;
9222                         break;
9223                 }
9224                 inot->initiator_id = event->initiator_id;
9225                 inot->sense_len = 0;
9226                 xpt_done((union ccb *)inot);
9227                 lstate->event_r_idx++;
9228                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9229                         lstate->event_r_idx = 0;
9230         }
9231 }
9232 #endif
9233
9234 /******************** Sequencer Program Patching/Download *********************/
9235
9236 #ifdef AHD_DUMP_SEQ
9237 void
9238 ahd_dumpseq(struct ahd_softc* ahd)
9239 {
9240         int i;
9241         int max_prog;
9242
9243         max_prog = 2048;
9244
9245         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9246         ahd_outw(ahd, PRGMCNT, 0);
9247         for (i = 0; i < max_prog; i++) {
9248                 uint8_t ins_bytes[4];
9249
9250                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
9251                 printk("0x%08x\n", ins_bytes[0] << 24
9252                                  | ins_bytes[1] << 16
9253                                  | ins_bytes[2] << 8
9254                                  | ins_bytes[3]);
9255         }
9256 }
9257 #endif
9258
9259 static void
9260 ahd_loadseq(struct ahd_softc *ahd)
9261 {
9262         struct  cs cs_table[NUM_CRITICAL_SECTIONS];
9263         u_int   begin_set[NUM_CRITICAL_SECTIONS];
9264         u_int   end_set[NUM_CRITICAL_SECTIONS];
9265         const struct patch *cur_patch;
9266         u_int   cs_count;
9267         u_int   cur_cs;
9268         u_int   i;
9269         int     downloaded;
9270         u_int   skip_addr;
9271         u_int   sg_prefetch_cnt;
9272         u_int   sg_prefetch_cnt_limit;
9273         u_int   sg_prefetch_align;
9274         u_int   sg_size;
9275         u_int   cacheline_mask;
9276         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
9277
9278         if (bootverbose)
9279                 printk("%s: Downloading Sequencer Program...",
9280                        ahd_name(ahd));
9281
9282 #if DOWNLOAD_CONST_COUNT != 8
9283 #error "Download Const Mismatch"
9284 #endif
9285         /*
9286          * Start out with 0 critical sections
9287          * that apply to this firmware load.
9288          */
9289         cs_count = 0;
9290         cur_cs = 0;
9291         memset(begin_set, 0, sizeof(begin_set));
9292         memset(end_set, 0, sizeof(end_set));
9293
9294         /*
9295          * Setup downloadable constant table.
9296          *
9297          * The computation for the S/G prefetch variables is
9298          * a bit complicated.  We would like to always fetch
9299          * in terms of cachelined sized increments.  However,
9300          * if the cacheline is not an even multiple of the
9301          * SG element size or is larger than our SG RAM, using
9302          * just the cache size might leave us with only a portion
9303          * of an SG element at the tail of a prefetch.  If the
9304          * cacheline is larger than our S/G prefetch buffer less
9305          * the size of an SG element, we may round down to a cacheline
9306          * that doesn't contain any or all of the S/G of interest
9307          * within the bounds of our S/G ram.  Provide variables to
9308          * the sequencer that will allow it to handle these edge
9309          * cases.
9310          */
9311         /* Start by aligning to the nearest cacheline. */
9312         sg_prefetch_align = ahd->pci_cachesize;
9313         if (sg_prefetch_align == 0)
9314                 sg_prefetch_align = 8;
9315         /* Round down to the nearest power of 2. */
9316         while (powerof2(sg_prefetch_align) == 0)
9317                 sg_prefetch_align--;
9318
9319         cacheline_mask = sg_prefetch_align - 1;
9320
9321         /*
9322          * If the cacheline boundary is greater than half our prefetch RAM
9323          * we risk not being able to fetch even a single complete S/G
9324          * segment if we align to that boundary.
9325          */
9326         if (sg_prefetch_align > CCSGADDR_MAX/2)
9327                 sg_prefetch_align = CCSGADDR_MAX/2;
9328         /* Start by fetching a single cacheline. */
9329         sg_prefetch_cnt = sg_prefetch_align;
9330         /*
9331          * Increment the prefetch count by cachelines until
9332          * at least one S/G element will fit.
9333          */
9334         sg_size = sizeof(struct ahd_dma_seg);
9335         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
9336                 sg_size = sizeof(struct ahd_dma64_seg);
9337         while (sg_prefetch_cnt < sg_size)
9338                 sg_prefetch_cnt += sg_prefetch_align;
9339         /*
9340          * If the cacheline is not an even multiple of
9341          * the S/G size, we may only get a partial S/G when
9342          * we align. Add a cacheline if this is the case.
9343          */
9344         if ((sg_prefetch_align % sg_size) != 0
9345          && (sg_prefetch_cnt < CCSGADDR_MAX))
9346                 sg_prefetch_cnt += sg_prefetch_align;
9347         /*
9348          * Lastly, compute a value that the sequencer can use
9349          * to determine if the remainder of the CCSGRAM buffer
9350          * has a full S/G element in it.
9351          */
9352         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
9353         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
9354         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
9355         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
9356         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
9357         download_consts[SG_SIZEOF] = sg_size;
9358         download_consts[PKT_OVERRUN_BUFOFFSET] =
9359                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
9360         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
9361         download_consts[CACHELINE_MASK] = cacheline_mask;
9362         cur_patch = patches;
9363         downloaded = 0;
9364         skip_addr = 0;
9365         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9366         ahd_outw(ahd, PRGMCNT, 0);
9367
9368         for (i = 0; i < sizeof(seqprog)/4; i++) {
9369                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
9370                         /*
9371                          * Don't download this instruction as it
9372                          * is in a patch that was removed.
9373                          */
9374                         continue;
9375                 }
9376                 /*
9377                  * Move through the CS table until we find a CS
9378                  * that might apply to this instruction.
9379                  */
9380                 for (; cur_cs < NUM_CRITICAL_SECTIONS; cur_cs++) {
9381                         if (critical_sections[cur_cs].end <= i) {
9382                                 if (begin_set[cs_count] == TRUE
9383                                  && end_set[cs_count] == FALSE) {
9384                                         cs_table[cs_count].end = downloaded;
9385                                         end_set[cs_count] = TRUE;
9386                                         cs_count++;
9387                                 }
9388                                 continue;
9389                         }
9390                         if (critical_sections[cur_cs].begin <= i
9391                          && begin_set[cs_count] == FALSE) {
9392                                 cs_table[cs_count].begin = downloaded;
9393                                 begin_set[cs_count] = TRUE;
9394                         }
9395                         break;
9396                 }
9397                 ahd_download_instr(ahd, i, download_consts);
9398                 downloaded++;
9399         }
9400
9401         ahd->num_critical_sections = cs_count;
9402         if (cs_count != 0) {
9403
9404                 cs_count *= sizeof(struct cs);
9405                 ahd->critical_sections = kmemdup(cs_table, cs_count, GFP_ATOMIC);
9406                 if (ahd->critical_sections == NULL)
9407                         panic("ahd_loadseq: Could not malloc");
9408         }
9409         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
9410
9411         if (bootverbose) {
9412                 printk(" %d instructions downloaded\n", downloaded);
9413                 printk("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
9414                        ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
9415         }
9416 }
9417
9418 static int
9419 ahd_check_patch(struct ahd_softc *ahd, const struct patch **start_patch,
9420                 u_int start_instr, u_int *skip_addr)
9421 {
9422         const struct patch *cur_patch;
9423         const struct patch *last_patch;
9424         u_int   num_patches;
9425
9426         num_patches = ARRAY_SIZE(patches);
9427         last_patch = &patches[num_patches];
9428         cur_patch = *start_patch;
9429
9430         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
9431
9432                 if (cur_patch->patch_func(ahd) == 0) {
9433
9434                         /* Start rejecting code */
9435                         *skip_addr = start_instr + cur_patch->skip_instr;
9436                         cur_patch += cur_patch->skip_patch;
9437                 } else {
9438                         /* Accepted this patch.  Advance to the next
9439                          * one and wait for our intruction pointer to
9440                          * hit this point.
9441                          */
9442                         cur_patch++;
9443                 }
9444         }
9445
9446         *start_patch = cur_patch;
9447         if (start_instr < *skip_addr)
9448                 /* Still skipping */
9449                 return (0);
9450
9451         return (1);
9452 }
9453
9454 static u_int
9455 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
9456 {
9457         const struct patch *cur_patch;
9458         int address_offset;
9459         u_int skip_addr;
9460         u_int i;
9461
9462         address_offset = 0;
9463         cur_patch = patches;
9464         skip_addr = 0;
9465
9466         for (i = 0; i < address;) {
9467
9468                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
9469
9470                 if (skip_addr > i) {
9471                         int end_addr;
9472
9473                         end_addr = min(address, skip_addr);
9474                         address_offset += end_addr - i;
9475                         i = skip_addr;
9476                 } else {
9477                         i++;
9478                 }
9479         }
9480         return (address - address_offset);
9481 }
9482
9483 static void
9484 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
9485 {
9486         union   ins_formats instr;
9487         struct  ins_format1 *fmt1_ins;
9488         struct  ins_format3 *fmt3_ins;
9489         u_int   opcode;
9490
9491         /*
9492          * The firmware is always compiled into a little endian format.
9493          */
9494         instr.integer = ahd_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
9495
9496         fmt1_ins = &instr.format1;
9497         fmt3_ins = NULL;
9498
9499         /* Pull the opcode */
9500         opcode = instr.format1.opcode;
9501         switch (opcode) {
9502         case AIC_OP_JMP:
9503         case AIC_OP_JC:
9504         case AIC_OP_JNC:
9505         case AIC_OP_CALL:
9506         case AIC_OP_JNE:
9507         case AIC_OP_JNZ:
9508         case AIC_OP_JE:
9509         case AIC_OP_JZ:
9510         {
9511                 fmt3_ins = &instr.format3;
9512                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
9513         }
9514                 fallthrough;
9515         case AIC_OP_OR:
9516         case AIC_OP_AND:
9517         case AIC_OP_XOR:
9518         case AIC_OP_ADD:
9519         case AIC_OP_ADC:
9520         case AIC_OP_BMOV:
9521                 if (fmt1_ins->parity != 0) {
9522                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
9523                 }
9524                 fmt1_ins->parity = 0;
9525                 fallthrough;
9526         case AIC_OP_ROL:
9527         {
9528                 int i, count;
9529
9530                 /* Calculate odd parity for the instruction */
9531                 for (i = 0, count = 0; i < 31; i++) {
9532                         uint32_t mask;
9533
9534                         mask = 0x01 << i;
9535                         if ((instr.integer & mask) != 0)
9536                                 count++;
9537                 }
9538                 if ((count & 0x01) == 0)
9539                         instr.format1.parity = 1;
9540
9541                 /* The sequencer is a little endian cpu */
9542                 instr.integer = ahd_htole32(instr.integer);
9543                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
9544                 break;
9545         }
9546         default:
9547                 panic("Unknown opcode encountered in seq program");
9548                 break;
9549         }
9550 }
9551
9552 static int
9553 ahd_probe_stack_size(struct ahd_softc *ahd)
9554 {
9555         int last_probe;
9556
9557         last_probe = 0;
9558         while (1) {
9559                 int i;
9560
9561                 /*
9562                  * We avoid using 0 as a pattern to avoid
9563                  * confusion if the stack implementation
9564                  * "back-fills" with zeros when "poping'
9565                  * entries.
9566                  */
9567                 for (i = 1; i <= last_probe+1; i++) {
9568                        ahd_outb(ahd, STACK, i & 0xFF);
9569                        ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
9570                 }
9571
9572                 /* Verify */
9573                 for (i = last_probe+1; i > 0; i--) {
9574                         u_int stack_entry;
9575
9576                         stack_entry = ahd_inb(ahd, STACK)
9577                                     |(ahd_inb(ahd, STACK) << 8);
9578                         if (stack_entry != i)
9579                                 goto sized;
9580                 }
9581                 last_probe++;
9582         }
9583 sized:
9584         return (last_probe);
9585 }
9586
9587 int
9588 ahd_print_register(const ahd_reg_parse_entry_t *table, u_int num_entries,
9589                    const char *name, u_int address, u_int value,
9590                    u_int *cur_column, u_int wrap_point)
9591 {
9592         int     printed;
9593         u_int   printed_mask;
9594
9595         if (cur_column != NULL && *cur_column >= wrap_point) {
9596                 printk("\n");
9597                 *cur_column = 0;
9598         }
9599         printed = printk("%s[0x%x]", name, value);
9600         if (table == NULL) {
9601                 printed += printk(" ");
9602                 *cur_column += printed;
9603                 return (printed);
9604         }
9605         printed_mask = 0;
9606         while (printed_mask != 0xFF) {
9607                 int entry;
9608
9609                 for (entry = 0; entry < num_entries; entry++) {
9610                         if (((value & table[entry].mask)
9611                           != table[entry].value)
9612                          || ((printed_mask & table[entry].mask)
9613                           == table[entry].mask))
9614                                 continue;
9615
9616                         printed += printk("%s%s",
9617                                           printed_mask == 0 ? ":(" : "|",
9618                                           table[entry].name);
9619                         printed_mask |= table[entry].mask;
9620
9621                         break;
9622                 }
9623                 if (entry >= num_entries)
9624                         break;
9625         }
9626         if (printed_mask != 0)
9627                 printed += printk(") ");
9628         else
9629                 printed += printk(" ");
9630         if (cur_column != NULL)
9631                 *cur_column += printed;
9632         return (printed);
9633 }
9634
9635 void
9636 ahd_dump_card_state(struct ahd_softc *ahd)
9637 {
9638         struct scb      *scb;
9639         ahd_mode_state   saved_modes;
9640         u_int            dffstat;
9641         int              paused;
9642         u_int            scb_index;
9643         u_int            saved_scb_index;
9644         u_int            cur_col;
9645         int              i;
9646
9647         if (ahd_is_paused(ahd)) {
9648                 paused = 1;
9649         } else {
9650                 paused = 0;
9651                 ahd_pause(ahd);
9652         }
9653         saved_modes = ahd_save_modes(ahd);
9654         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9655         printk(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
9656                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
9657                ahd_name(ahd),
9658                ahd_inw(ahd, CURADDR),
9659                ahd_build_mode_state(ahd, ahd->saved_src_mode,
9660                                     ahd->saved_dst_mode));
9661         if (paused)
9662                 printk("Card was paused\n");
9663
9664         if (ahd_check_cmdcmpltqueues(ahd))
9665                 printk("Completions are pending\n");
9666
9667         /*
9668          * Mode independent registers.
9669          */
9670         cur_col = 0;
9671         ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
9672         ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
9673         ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
9674         ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
9675         ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
9676         ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
9677         ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
9678         ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
9679         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
9680         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
9681         ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
9682         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
9683         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
9684         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
9685         ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
9686         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
9687         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
9688         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
9689         ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
9690         ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
9691                                        &cur_col, 50);
9692         ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
9693         ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
9694                                     &cur_col, 50);
9695         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
9696         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
9697         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
9698         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
9699         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
9700         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
9701         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
9702         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
9703         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
9704         ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
9705         ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
9706         ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
9707         printk("\n");
9708         printk("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
9709                "CURRSCB 0x%x NEXTSCB 0x%x\n",
9710                ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
9711                ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
9712                ahd_inw(ahd, NEXTSCB));
9713         cur_col = 0;
9714         /* QINFIFO */
9715         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
9716                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
9717                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
9718         saved_scb_index = ahd_get_scbptr(ahd);
9719         printk("Pending list:");
9720         i = 0;
9721         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9722                 if (i++ > AHD_SCB_MAX)
9723                         break;
9724                 cur_col = printk("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
9725                                  ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
9726                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9727                 ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
9728                                       &cur_col, 60);
9729                 ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
9730                                      &cur_col, 60);
9731         }
9732         printk("\nTotal %d\n", i);
9733
9734         printk("Kernel Free SCB list: ");
9735         i = 0;
9736         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
9737                 struct scb *list_scb;
9738
9739                 list_scb = scb;
9740                 do {
9741                         printk("%d ", SCB_GET_TAG(list_scb));
9742                         list_scb = LIST_NEXT(list_scb, collision_links);
9743                 } while (list_scb && i++ < AHD_SCB_MAX);
9744         }
9745
9746         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
9747                 if (i++ > AHD_SCB_MAX)
9748                         break;
9749                 printk("%d ", SCB_GET_TAG(scb));
9750         }
9751         printk("\n");
9752
9753         printk("Sequencer Complete DMA-inprog list: ");
9754         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9755         i = 0;
9756         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9757                 ahd_set_scbptr(ahd, scb_index);
9758                 printk("%d ", scb_index);
9759                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9760         }
9761         printk("\n");
9762
9763         printk("Sequencer Complete list: ");
9764         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9765         i = 0;
9766         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9767                 ahd_set_scbptr(ahd, scb_index);
9768                 printk("%d ", scb_index);
9769                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9770         }
9771         printk("\n");
9772
9773
9774         printk("Sequencer DMA-Up and Complete list: ");
9775         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9776         i = 0;
9777         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9778                 ahd_set_scbptr(ahd, scb_index);
9779                 printk("%d ", scb_index);
9780                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9781         }
9782         printk("\n");
9783         printk("Sequencer On QFreeze and Complete list: ");
9784         scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9785         i = 0;
9786         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9787                 ahd_set_scbptr(ahd, scb_index);
9788                 printk("%d ", scb_index);
9789                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9790         }
9791         printk("\n");
9792         ahd_set_scbptr(ahd, saved_scb_index);
9793         dffstat = ahd_inb(ahd, DFFSTAT);
9794         for (i = 0; i < 2; i++) {
9795 #ifdef AHD_DEBUG
9796                 struct scb *fifo_scb;
9797 #endif
9798                 u_int       fifo_scbptr;
9799
9800                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9801                 fifo_scbptr = ahd_get_scbptr(ahd);
9802                 printk("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9803                        ahd_name(ahd), i,
9804                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9805                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9806                 cur_col = 0;
9807                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9808                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9809                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9810                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9811                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9812                                           &cur_col, 50);
9813                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9814                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9815                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9816                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9817                 if (cur_col > 50) {
9818                         printk("\n");
9819                         cur_col = 0;
9820                 }
9821                 cur_col += printk("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9822                                   ahd_inl(ahd, SHADDR+4),
9823                                   ahd_inl(ahd, SHADDR),
9824                                   (ahd_inb(ahd, SHCNT)
9825                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
9826                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
9827                 if (cur_col > 50) {
9828                         printk("\n");
9829                         cur_col = 0;
9830                 }
9831                 cur_col += printk("HADDR = 0x%x%x, HCNT = 0x%x ",
9832                                   ahd_inl(ahd, HADDR+4),
9833                                   ahd_inl(ahd, HADDR),
9834                                   (ahd_inb(ahd, HCNT)
9835                                 | (ahd_inb(ahd, HCNT + 1) << 8)
9836                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
9837                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9838 #ifdef AHD_DEBUG
9839                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
9840                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9841                         if (fifo_scb != NULL)
9842                                 ahd_dump_sglist(fifo_scb);
9843                 }
9844 #endif
9845         }
9846         printk("\nLQIN: ");
9847         for (i = 0; i < 20; i++)
9848                 printk("0x%x ", ahd_inb(ahd, LQIN + i));
9849         printk("\n");
9850         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9851         printk("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9852                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9853                ahd_inb(ahd, OPTIONMODE));
9854         printk("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9855                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9856                ahd_inb(ahd, MAXCMDCNT));
9857         printk("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9858                ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9859                ahd_inb(ahd, SAVED_LUN));
9860         ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9861         printk("\n");
9862         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9863         cur_col = 0;
9864         ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9865         printk("\n");
9866         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9867         printk("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9868                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9869                ahd_inw(ahd, DINDEX));
9870         printk("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9871                ahd_name(ahd), ahd_get_scbptr(ahd),
9872                ahd_inw_scbram(ahd, SCB_NEXT),
9873                ahd_inw_scbram(ahd, SCB_NEXT2));
9874         printk("CDB %x %x %x %x %x %x\n",
9875                ahd_inb_scbram(ahd, SCB_CDB_STORE),
9876                ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9877                ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9878                ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9879                ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9880                ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9881         printk("STACK:");
9882         for (i = 0; i < ahd->stack_size; i++) {
9883                 ahd->saved_stack[i] =
9884                     ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9885                 printk(" 0x%x", ahd->saved_stack[i]);
9886         }
9887         for (i = ahd->stack_size-1; i >= 0; i--) {
9888                 ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9889                 ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9890         }
9891         printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9892         ahd_restore_modes(ahd, saved_modes);
9893         if (paused == 0)
9894                 ahd_unpause(ahd);
9895 }
9896
9897 #if 0
9898 void
9899 ahd_dump_scbs(struct ahd_softc *ahd)
9900 {
9901         ahd_mode_state saved_modes;
9902         u_int          saved_scb_index;
9903         int            i;
9904
9905         saved_modes = ahd_save_modes(ahd);
9906         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9907         saved_scb_index = ahd_get_scbptr(ahd);
9908         for (i = 0; i < AHD_SCB_MAX; i++) {
9909                 ahd_set_scbptr(ahd, i);
9910                 printk("%3d", i);
9911                 printk("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9912                        ahd_inb_scbram(ahd, SCB_CONTROL),
9913                        ahd_inb_scbram(ahd, SCB_SCSIID),
9914                        ahd_inw_scbram(ahd, SCB_NEXT),
9915                        ahd_inw_scbram(ahd, SCB_NEXT2),
9916                        ahd_inl_scbram(ahd, SCB_SGPTR),
9917                        ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9918         }
9919         printk("\n");
9920         ahd_set_scbptr(ahd, saved_scb_index);
9921         ahd_restore_modes(ahd, saved_modes);
9922 }
9923 #endif  /*  0  */
9924
9925 /**************************** Flexport Logic **********************************/
9926 /*
9927  * Read count 16bit words from 16bit word address start_addr from the
9928  * SEEPROM attached to the controller, into buf, using the controller's
9929  * SEEPROM reading state machine.  Optionally treat the data as a byte
9930  * stream in terms of byte order.
9931  */
9932 int
9933 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9934                  u_int start_addr, u_int count, int bytestream)
9935 {
9936         u_int cur_addr;
9937         u_int end_addr;
9938         int   error;
9939
9940         /*
9941          * If we never make it through the loop even once,
9942          * we were passed invalid arguments.
9943          */
9944         error = EINVAL;
9945         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9946         end_addr = start_addr + count;
9947         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9948
9949                 ahd_outb(ahd, SEEADR, cur_addr);
9950                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
9951
9952                 error = ahd_wait_seeprom(ahd);
9953                 if (error)
9954                         break;
9955                 if (bytestream != 0) {
9956                         uint8_t *bytestream_ptr;
9957
9958                         bytestream_ptr = (uint8_t *)buf;
9959                         *bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
9960                         *bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
9961                 } else {
9962                         /*
9963                          * ahd_inw() already handles machine byte order.
9964                          */
9965                         *buf = ahd_inw(ahd, SEEDAT);
9966                 }
9967                 buf++;
9968         }
9969         return (error);
9970 }
9971
9972 /*
9973  * Write count 16bit words from buf, into SEEPROM attache to the
9974  * controller starting at 16bit word address start_addr, using the
9975  * controller's SEEPROM writing state machine.
9976  */
9977 int
9978 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9979                   u_int start_addr, u_int count)
9980 {
9981         u_int cur_addr;
9982         u_int end_addr;
9983         int   error;
9984         int   retval;
9985
9986         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9987         error = ENOENT;
9988
9989         /* Place the chip into write-enable mode */
9990         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
9991         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
9992         error = ahd_wait_seeprom(ahd);
9993         if (error)
9994                 return (error);
9995
9996         /*
9997          * Write the data.  If we don't get through the loop at
9998          * least once, the arguments were invalid.
9999          */
10000         retval = EINVAL;
10001         end_addr = start_addr + count;
10002         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
10003                 ahd_outw(ahd, SEEDAT, *buf++);
10004                 ahd_outb(ahd, SEEADR, cur_addr);
10005                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
10006
10007                 retval = ahd_wait_seeprom(ahd);
10008                 if (retval)
10009                         break;
10010         }
10011
10012         /*
10013          * Disable writes.
10014          */
10015         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
10016         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
10017         error = ahd_wait_seeprom(ahd);
10018         if (error)
10019                 return (error);
10020         return (retval);
10021 }
10022
10023 /*
10024  * Wait ~100us for the serial eeprom to satisfy our request.
10025  */
10026 static int
10027 ahd_wait_seeprom(struct ahd_softc *ahd)
10028 {
10029         int cnt;
10030
10031         cnt = 5000;
10032         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
10033                 ahd_delay(5);
10034
10035         if (cnt == 0)
10036                 return (ETIMEDOUT);
10037         return (0);
10038 }
10039
10040 /*
10041  * Validate the two checksums in the per_channel
10042  * vital product data struct.
10043  */
10044 static int
10045 ahd_verify_vpd_cksum(struct vpd_config *vpd)
10046 {
10047         int i;
10048         int maxaddr;
10049         uint32_t checksum;
10050         uint8_t *vpdarray;
10051
10052         vpdarray = (uint8_t *)vpd;
10053         maxaddr = offsetof(struct vpd_config, vpd_checksum);
10054         checksum = 0;
10055         for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
10056                 checksum = checksum + vpdarray[i];
10057         if (checksum == 0
10058          || (-checksum & 0xFF) != vpd->vpd_checksum)
10059                 return (0);
10060
10061         checksum = 0;
10062         maxaddr = offsetof(struct vpd_config, checksum);
10063         for (i = offsetof(struct vpd_config, default_target_flags);
10064              i < maxaddr; i++)
10065                 checksum = checksum + vpdarray[i];
10066         if (checksum == 0
10067          || (-checksum & 0xFF) != vpd->checksum)
10068                 return (0);
10069         return (1);
10070 }
10071
10072 int
10073 ahd_verify_cksum(struct seeprom_config *sc)
10074 {
10075         int i;
10076         int maxaddr;
10077         uint32_t checksum;
10078         uint16_t *scarray;
10079
10080         maxaddr = (sizeof(*sc)/2) - 1;
10081         checksum = 0;
10082         scarray = (uint16_t *)sc;
10083
10084         for (i = 0; i < maxaddr; i++)
10085                 checksum = checksum + scarray[i];
10086         if (checksum == 0
10087          || (checksum & 0xFFFF) != sc->checksum) {
10088                 return (0);
10089         } else {
10090                 return (1);
10091         }
10092 }
10093
10094 int
10095 ahd_acquire_seeprom(struct ahd_softc *ahd)
10096 {
10097         /*
10098          * We should be able to determine the SEEPROM type
10099          * from the flexport logic, but unfortunately not
10100          * all implementations have this logic and there is
10101          * no programatic method for determining if the logic
10102          * is present.
10103          */
10104         return (1);
10105 #if 0
10106         uint8_t seetype;
10107         int     error;
10108
10109         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
10110         if (error != 0
10111          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
10112                 return (0);
10113         return (1);
10114 #endif
10115 }
10116
10117 void
10118 ahd_release_seeprom(struct ahd_softc *ahd)
10119 {
10120         /* Currently a no-op */
10121 }
10122
10123 /*
10124  * Wait at most 2 seconds for flexport arbitration to succeed.
10125  */
10126 static int
10127 ahd_wait_flexport(struct ahd_softc *ahd)
10128 {
10129         int cnt;
10130
10131         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10132         cnt = 1000000 * 2 / 5;
10133         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
10134                 ahd_delay(5);
10135
10136         if (cnt == 0)
10137                 return (ETIMEDOUT);
10138         return (0);
10139 }
10140
10141 int
10142 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
10143 {
10144         int error;
10145
10146         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10147         if (addr > 7)
10148                 panic("ahd_write_flexport: address out of range");
10149         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10150         error = ahd_wait_flexport(ahd);
10151         if (error != 0)
10152                 return (error);
10153         ahd_outb(ahd, BRDDAT, value);
10154         ahd_flush_device_writes(ahd);
10155         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
10156         ahd_flush_device_writes(ahd);
10157         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10158         ahd_flush_device_writes(ahd);
10159         ahd_outb(ahd, BRDCTL, 0);
10160         ahd_flush_device_writes(ahd);
10161         return (0);
10162 }
10163
10164 int
10165 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
10166 {
10167         int     error;
10168
10169         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10170         if (addr > 7)
10171                 panic("ahd_read_flexport: address out of range");
10172         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
10173         error = ahd_wait_flexport(ahd);
10174         if (error != 0)
10175                 return (error);
10176         *value = ahd_inb(ahd, BRDDAT);
10177         ahd_outb(ahd, BRDCTL, 0);
10178         ahd_flush_device_writes(ahd);
10179         return (0);
10180 }
10181
10182 /************************* Target Mode ****************************************/
10183 #ifdef AHD_TARGET_MODE
10184 cam_status
10185 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
10186                     struct ahd_tmode_tstate **tstate,
10187                     struct ahd_tmode_lstate **lstate,
10188                     int notfound_failure)
10189 {
10190
10191         if ((ahd->features & AHD_TARGETMODE) == 0)
10192                 return (CAM_REQ_INVALID);
10193
10194         /*
10195          * Handle the 'black hole' device that sucks up
10196          * requests to unattached luns on enabled targets.
10197          */
10198         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
10199          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
10200                 *tstate = NULL;
10201                 *lstate = ahd->black_hole;
10202         } else {
10203                 u_int max_id;
10204
10205                 max_id = (ahd->features & AHD_WIDE) ? 16 : 8;
10206                 if (ccb->ccb_h.target_id >= max_id)
10207                         return (CAM_TID_INVALID);
10208
10209                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
10210                         return (CAM_LUN_INVALID);
10211
10212                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
10213                 *lstate = NULL;
10214                 if (*tstate != NULL)
10215                         *lstate =
10216                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
10217         }
10218
10219         if (notfound_failure != 0 && *lstate == NULL)
10220                 return (CAM_PATH_INVALID);
10221
10222         return (CAM_REQ_CMP);
10223 }
10224
10225 void
10226 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
10227 {
10228 #if NOT_YET
10229         struct     ahd_tmode_tstate *tstate;
10230         struct     ahd_tmode_lstate *lstate;
10231         struct     ccb_en_lun *cel;
10232         cam_status status;
10233         u_int      target;
10234         u_int      lun;
10235         u_int      target_mask;
10236         u_long     s;
10237         char       channel;
10238
10239         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
10240                                      /*notfound_failure*/FALSE);
10241
10242         if (status != CAM_REQ_CMP) {
10243                 ccb->ccb_h.status = status;
10244                 return;
10245         }
10246
10247         if ((ahd->features & AHD_MULTIROLE) != 0) {
10248                 u_int      our_id;
10249
10250                 our_id = ahd->our_id;
10251                 if (ccb->ccb_h.target_id != our_id) {
10252                         if ((ahd->features & AHD_MULTI_TID) != 0
10253                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
10254                                 /*
10255                                  * Only allow additional targets if
10256                                  * the initiator role is disabled.
10257                                  * The hardware cannot handle a re-select-in
10258                                  * on the initiator id during a re-select-out
10259                                  * on a different target id.
10260                                  */
10261                                 status = CAM_TID_INVALID;
10262                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
10263                                 || ahd->enabled_luns > 0) {
10264                                 /*
10265                                  * Only allow our target id to change
10266                                  * if the initiator role is not configured
10267                                  * and there are no enabled luns which
10268                                  * are attached to the currently registered
10269                                  * scsi id.
10270                                  */
10271                                 status = CAM_TID_INVALID;
10272                         }
10273                 }
10274         }
10275
10276         if (status != CAM_REQ_CMP) {
10277                 ccb->ccb_h.status = status;
10278                 return;
10279         }
10280
10281         /*
10282          * We now have an id that is valid.
10283          * If we aren't in target mode, switch modes.
10284          */
10285         if ((ahd->flags & AHD_TARGETROLE) == 0
10286          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
10287                 u_long  s;
10288
10289                 printk("Configuring Target Mode\n");
10290                 ahd_lock(ahd, &s);
10291                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
10292                         ccb->ccb_h.status = CAM_BUSY;
10293                         ahd_unlock(ahd, &s);
10294                         return;
10295                 }
10296                 ahd->flags |= AHD_TARGETROLE;
10297                 if ((ahd->features & AHD_MULTIROLE) == 0)
10298                         ahd->flags &= ~AHD_INITIATORROLE;
10299                 ahd_pause(ahd);
10300                 ahd_loadseq(ahd);
10301                 ahd_restart(ahd);
10302                 ahd_unlock(ahd, &s);
10303         }
10304         cel = &ccb->cel;
10305         target = ccb->ccb_h.target_id;
10306         lun = ccb->ccb_h.target_lun;
10307         channel = SIM_CHANNEL(ahd, sim);
10308         target_mask = 0x01 << target;
10309         if (channel == 'B')
10310                 target_mask <<= 8;
10311
10312         if (cel->enable != 0) {
10313                 u_int scsiseq1;
10314
10315                 /* Are we already enabled?? */
10316                 if (lstate != NULL) {
10317                         xpt_print_path(ccb->ccb_h.path);
10318                         printk("Lun already enabled\n");
10319                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
10320                         return;
10321                 }
10322
10323                 if (cel->grp6_len != 0
10324                  || cel->grp7_len != 0) {
10325                         /*
10326                          * Don't (yet?) support vendor
10327                          * specific commands.
10328                          */
10329                         ccb->ccb_h.status = CAM_REQ_INVALID;
10330                         printk("Non-zero Group Codes\n");
10331                         return;
10332                 }
10333
10334                 /*
10335                  * Seems to be okay.
10336                  * Setup our data structures.
10337                  */
10338                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
10339                         tstate = ahd_alloc_tstate(ahd, target, channel);
10340                         if (tstate == NULL) {
10341                                 xpt_print_path(ccb->ccb_h.path);
10342                                 printk("Couldn't allocate tstate\n");
10343                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10344                                 return;
10345                         }
10346                 }
10347                 lstate = kzalloc(sizeof(*lstate), GFP_ATOMIC);
10348                 if (lstate == NULL) {
10349                         xpt_print_path(ccb->ccb_h.path);
10350                         printk("Couldn't allocate lstate\n");
10351                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10352                         return;
10353                 }
10354                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
10355                                          xpt_path_path_id(ccb->ccb_h.path),
10356                                          xpt_path_target_id(ccb->ccb_h.path),
10357                                          xpt_path_lun_id(ccb->ccb_h.path));
10358                 if (status != CAM_REQ_CMP) {
10359                         kfree(lstate);
10360                         xpt_print_path(ccb->ccb_h.path);
10361                         printk("Couldn't allocate path\n");
10362                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10363                         return;
10364                 }
10365                 SLIST_INIT(&lstate->accept_tios);
10366                 SLIST_INIT(&lstate->immed_notifies);
10367                 ahd_lock(ahd, &s);
10368                 ahd_pause(ahd);
10369                 if (target != CAM_TARGET_WILDCARD) {
10370                         tstate->enabled_luns[lun] = lstate;
10371                         ahd->enabled_luns++;
10372
10373                         if ((ahd->features & AHD_MULTI_TID) != 0) {
10374                                 u_int targid_mask;
10375
10376                                 targid_mask = ahd_inw(ahd, TARGID);
10377                                 targid_mask |= target_mask;
10378                                 ahd_outw(ahd, TARGID, targid_mask);
10379                                 ahd_update_scsiid(ahd, targid_mask);
10380                         } else {
10381                                 u_int our_id;
10382                                 char  channel;
10383
10384                                 channel = SIM_CHANNEL(ahd, sim);
10385                                 our_id = SIM_SCSI_ID(ahd, sim);
10386
10387                                 /*
10388                                  * This can only happen if selections
10389                                  * are not enabled
10390                                  */
10391                                 if (target != our_id) {
10392                                         u_int sblkctl;
10393                                         char  cur_channel;
10394                                         int   swap;
10395
10396                                         sblkctl = ahd_inb(ahd, SBLKCTL);
10397                                         cur_channel = (sblkctl & SELBUSB)
10398                                                     ? 'B' : 'A';
10399                                         if ((ahd->features & AHD_TWIN) == 0)
10400                                                 cur_channel = 'A';
10401                                         swap = cur_channel != channel;
10402                                         ahd->our_id = target;
10403
10404                                         if (swap)
10405                                                 ahd_outb(ahd, SBLKCTL,
10406                                                          sblkctl ^ SELBUSB);
10407
10408                                         ahd_outb(ahd, SCSIID, target);
10409
10410                                         if (swap)
10411                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
10412                                 }
10413                         }
10414                 } else
10415                         ahd->black_hole = lstate;
10416                 /* Allow select-in operations */
10417                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
10418                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10419                         scsiseq1 |= ENSELI;
10420                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10421                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10422                         scsiseq1 |= ENSELI;
10423                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10424                 }
10425                 ahd_unpause(ahd);
10426                 ahd_unlock(ahd, &s);
10427                 ccb->ccb_h.status = CAM_REQ_CMP;
10428                 xpt_print_path(ccb->ccb_h.path);
10429                 printk("Lun now enabled for target mode\n");
10430         } else {
10431                 struct scb *scb;
10432                 int i, empty;
10433
10434                 if (lstate == NULL) {
10435                         ccb->ccb_h.status = CAM_LUN_INVALID;
10436                         return;
10437                 }
10438
10439                 ahd_lock(ahd, &s);
10440
10441                 ccb->ccb_h.status = CAM_REQ_CMP;
10442                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
10443                         struct ccb_hdr *ccbh;
10444
10445                         ccbh = &scb->io_ctx->ccb_h;
10446                         if (ccbh->func_code == XPT_CONT_TARGET_IO
10447                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
10448                                 printk("CTIO pending\n");
10449                                 ccb->ccb_h.status = CAM_REQ_INVALID;
10450                                 ahd_unlock(ahd, &s);
10451                                 return;
10452                         }
10453                 }
10454
10455                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
10456                         printk("ATIOs pending\n");
10457                         ccb->ccb_h.status = CAM_REQ_INVALID;
10458                 }
10459
10460                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
10461                         printk("INOTs pending\n");
10462                         ccb->ccb_h.status = CAM_REQ_INVALID;
10463                 }
10464
10465                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
10466                         ahd_unlock(ahd, &s);
10467                         return;
10468                 }
10469
10470                 xpt_print_path(ccb->ccb_h.path);
10471                 printk("Target mode disabled\n");
10472                 xpt_free_path(lstate->path);
10473                 kfree(lstate);
10474
10475                 ahd_pause(ahd);
10476                 /* Can we clean up the target too? */
10477                 if (target != CAM_TARGET_WILDCARD) {
10478                         tstate->enabled_luns[lun] = NULL;
10479                         ahd->enabled_luns--;
10480                         for (empty = 1, i = 0; i < 8; i++)
10481                                 if (tstate->enabled_luns[i] != NULL) {
10482                                         empty = 0;
10483                                         break;
10484                                 }
10485
10486                         if (empty) {
10487                                 ahd_free_tstate(ahd, target, channel,
10488                                                 /*force*/FALSE);
10489                                 if (ahd->features & AHD_MULTI_TID) {
10490                                         u_int targid_mask;
10491
10492                                         targid_mask = ahd_inw(ahd, TARGID);
10493                                         targid_mask &= ~target_mask;
10494                                         ahd_outw(ahd, TARGID, targid_mask);
10495                                         ahd_update_scsiid(ahd, targid_mask);
10496                                 }
10497                         }
10498                 } else {
10499
10500                         ahd->black_hole = NULL;
10501
10502                         /*
10503                          * We can't allow selections without
10504                          * our black hole device.
10505                          */
10506                         empty = TRUE;
10507                 }
10508                 if (ahd->enabled_luns == 0) {
10509                         /* Disallow select-in */
10510                         u_int scsiseq1;
10511
10512                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10513                         scsiseq1 &= ~ENSELI;
10514                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10515                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10516                         scsiseq1 &= ~ENSELI;
10517                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10518
10519                         if ((ahd->features & AHD_MULTIROLE) == 0) {
10520                                 printk("Configuring Initiator Mode\n");
10521                                 ahd->flags &= ~AHD_TARGETROLE;
10522                                 ahd->flags |= AHD_INITIATORROLE;
10523                                 ahd_pause(ahd);
10524                                 ahd_loadseq(ahd);
10525                                 ahd_restart(ahd);
10526                                 /*
10527                                  * Unpaused.  The extra unpause
10528                                  * that follows is harmless.
10529                                  */
10530                         }
10531                 }
10532                 ahd_unpause(ahd);
10533                 ahd_unlock(ahd, &s);
10534         }
10535 #endif
10536 }
10537
10538 static void
10539 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
10540 {
10541 #if NOT_YET
10542         u_int scsiid_mask;
10543         u_int scsiid;
10544
10545         if ((ahd->features & AHD_MULTI_TID) == 0)
10546                 panic("ahd_update_scsiid called on non-multitid unit\n");
10547
10548         /*
10549          * Since we will rely on the TARGID mask
10550          * for selection enables, ensure that OID
10551          * in SCSIID is not set to some other ID
10552          * that we don't want to allow selections on.
10553          */
10554         if ((ahd->features & AHD_ULTRA2) != 0)
10555                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
10556         else
10557                 scsiid = ahd_inb(ahd, SCSIID);
10558         scsiid_mask = 0x1 << (scsiid & OID);
10559         if ((targid_mask & scsiid_mask) == 0) {
10560                 u_int our_id;
10561
10562                 /* ffs counts from 1 */
10563                 our_id = ffs(targid_mask);
10564                 if (our_id == 0)
10565                         our_id = ahd->our_id;
10566                 else
10567                         our_id--;
10568                 scsiid &= TID;
10569                 scsiid |= our_id;
10570         }
10571         if ((ahd->features & AHD_ULTRA2) != 0)
10572                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
10573         else
10574                 ahd_outb(ahd, SCSIID, scsiid);
10575 #endif
10576 }
10577
10578 static void
10579 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
10580 {
10581         struct target_cmd *cmd;
10582
10583         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
10584         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
10585
10586                 /*
10587                  * Only advance through the queue if we
10588                  * have the resources to process the command.
10589                  */
10590                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
10591                         break;
10592
10593                 cmd->cmd_valid = 0;
10594                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
10595                                 ahd->shared_data_map.dmamap,
10596                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
10597                                 sizeof(struct target_cmd),
10598                                 BUS_DMASYNC_PREREAD);
10599                 ahd->tqinfifonext++;
10600
10601                 /*
10602                  * Lazily update our position in the target mode incoming
10603                  * command queue as seen by the sequencer.
10604                  */
10605                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
10606                         u_int hs_mailbox;
10607
10608                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
10609                         hs_mailbox &= ~HOST_TQINPOS;
10610                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
10611                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
10612                 }
10613         }
10614 }
10615
10616 static int
10617 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
10618 {
10619         struct    ahd_tmode_tstate *tstate;
10620         struct    ahd_tmode_lstate *lstate;
10621         struct    ccb_accept_tio *atio;
10622         uint8_t *byte;
10623         int       initiator;
10624         int       target;
10625         int       lun;
10626
10627         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
10628         target = SCSIID_OUR_ID(cmd->scsiid);
10629         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
10630
10631         byte = cmd->bytes;
10632         tstate = ahd->enabled_targets[target];
10633         lstate = NULL;
10634         if (tstate != NULL)
10635                 lstate = tstate->enabled_luns[lun];
10636
10637         /*
10638          * Commands for disabled luns go to the black hole driver.
10639          */
10640         if (lstate == NULL)
10641                 lstate = ahd->black_hole;
10642
10643         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
10644         if (atio == NULL) {
10645                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
10646                 /*
10647                  * Wait for more ATIOs from the peripheral driver for this lun.
10648                  */
10649                 return (1);
10650         } else
10651                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
10652 #ifdef AHD_DEBUG
10653         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10654                 printk("Incoming command from %d for %d:%d%s\n",
10655                        initiator, target, lun,
10656                        lstate == ahd->black_hole ? "(Black Holed)" : "");
10657 #endif
10658         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
10659
10660         if (lstate == ahd->black_hole) {
10661                 /* Fill in the wildcards */
10662                 atio->ccb_h.target_id = target;
10663                 atio->ccb_h.target_lun = lun;
10664         }
10665
10666         /*
10667          * Package it up and send it off to
10668          * whomever has this lun enabled.
10669          */
10670         atio->sense_len = 0;
10671         atio->init_id = initiator;
10672         if (byte[0] != 0xFF) {
10673                 /* Tag was included */
10674                 atio->tag_action = *byte++;
10675                 atio->tag_id = *byte++;
10676                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
10677         } else {
10678                 atio->ccb_h.flags = 0;
10679         }
10680         byte++;
10681
10682         /* Okay.  Now determine the cdb size based on the command code */
10683         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
10684         case 0:
10685                 atio->cdb_len = 6;
10686                 break;
10687         case 1:
10688         case 2:
10689                 atio->cdb_len = 10;
10690                 break;
10691         case 4:
10692                 atio->cdb_len = 16;
10693                 break;
10694         case 5:
10695                 atio->cdb_len = 12;
10696                 break;
10697         case 3:
10698         default:
10699                 /* Only copy the opcode. */
10700                 atio->cdb_len = 1;
10701                 printk("Reserved or VU command code type encountered\n");
10702                 break;
10703         }
10704
10705         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
10706
10707         atio->ccb_h.status |= CAM_CDB_RECVD;
10708
10709         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
10710                 /*
10711                  * We weren't allowed to disconnect.
10712                  * We're hanging on the bus until a
10713                  * continue target I/O comes in response
10714                  * to this accept tio.
10715                  */
10716 #ifdef AHD_DEBUG
10717                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10718                         printk("Received Immediate Command %d:%d:%d - %p\n",
10719                                initiator, target, lun, ahd->pending_device);
10720 #endif
10721                 ahd->pending_device = lstate;
10722                 ahd_freeze_ccb((union ccb *)atio);
10723                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
10724         }
10725         xpt_done((union ccb*)atio);
10726         return (0);
10727 }
10728
10729 #endif