iPXE
qib7322.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdint.h>
00027 #include <stdlib.h>
00028 #include <errno.h>
00029 #include <unistd.h>
00030 #include <assert.h>
00031 #include <ipxe/io.h>
00032 #include <ipxe/pci.h>
00033 #include <ipxe/infiniband.h>
00034 #include <ipxe/i2c.h>
00035 #include <ipxe/bitbash.h>
00036 #include <ipxe/malloc.h>
00037 #include <ipxe/iobuf.h>
00038 #include <ipxe/pcibackup.h>
00039 #include "qib7322.h"
00040 
00041 /**
00042  * @file
00043  *
00044  * QLogic QIB7322 Infiniband HCA
00045  *
00046  */
00047 
00048 /** A QIB7322 send buffer set */
00049 struct qib7322_send_buffers {
00050         /** Offset within register space of the first send buffer */
00051         unsigned long base;
00052         /** Send buffer size */
00053         unsigned int size;
00054         /** Index of first send buffer */
00055         unsigned int start;
00056         /** Number of send buffers
00057          *
00058          * Must be a power of two.
00059          */
00060         unsigned int count;
00061         /** Send buffer availability producer counter */
00062         unsigned int prod;
00063         /** Send buffer availability consumer counter */
00064         unsigned int cons;
00065         /** Send buffer availability */
00066         uint16_t avail[0];
00067 };
00068 
00069 /** A QIB7322 send work queue */
00070 struct qib7322_send_work_queue {
00071         /** Send buffer set */
00072         struct qib7322_send_buffers *send_bufs;
00073         /** Send buffer usage */
00074         uint16_t *used;
00075         /** Producer index */
00076         unsigned int prod;
00077         /** Consumer index */
00078         unsigned int cons;
00079 };
00080 
00081 /** A QIB7322 receive work queue */
00082 struct qib7322_recv_work_queue {
00083         /** Receive header ring */
00084         void *header;
00085         /** Receive header producer offset (written by hardware) */
00086         struct QIB_7322_scalar header_prod;
00087         /** Receive header consumer offset */
00088         unsigned int header_cons;
00089         /** Offset within register space of the eager array */
00090         unsigned long eager_array;
00091         /** Number of entries in eager array */
00092         unsigned int eager_entries;
00093         /** Eager array producer index */
00094         unsigned int eager_prod;
00095         /** Eager array consumer index */
00096         unsigned int eager_cons;
00097 };
00098 
00099 /** A QIB7322 HCA */
00100 struct qib7322 {
00101         /** Registers */
00102         void *regs;
00103 
00104         /** In-use contexts */
00105         uint8_t used_ctx[QIB7322_NUM_CONTEXTS];
00106         /** Send work queues */
00107         struct qib7322_send_work_queue send_wq[QIB7322_NUM_CONTEXTS];
00108         /** Receive work queues */
00109         struct qib7322_recv_work_queue recv_wq[QIB7322_NUM_CONTEXTS];
00110 
00111         /** Send buffer availability (reported by hardware) */
00112         struct QIB_7322_SendBufAvail *sendbufavail;
00113         /** Small send buffers */
00114         struct qib7322_send_buffers *send_bufs_small;
00115         /** VL15 port 0 send buffers */
00116         struct qib7322_send_buffers *send_bufs_vl15_port0;
00117         /** VL15 port 1 send buffers */
00118         struct qib7322_send_buffers *send_bufs_vl15_port1;
00119 
00120         /** I2C bit-bashing interface */
00121         struct i2c_bit_basher i2c;
00122         /** I2C serial EEPROM */
00123         struct i2c_device eeprom;
00124 
00125         /** Base GUID */
00126         union ib_guid guid;
00127         /** Infiniband devices */
00128         struct ib_device *ibdev[QIB7322_MAX_PORTS];
00129 };
00130 
00131 /***************************************************************************
00132  *
00133  * QIB7322 register access
00134  *
00135  ***************************************************************************
00136  *
00137  * This card requires atomic 64-bit accesses.  Strange things happen
00138  * if you try to use 32-bit accesses; sometimes they work, sometimes
00139  * they don't, sometimes you get random data.
00140  */
00141 
00142 /**
00143  * Read QIB7322 qword register
00144  *
00145  * @v qib7322           QIB7322 device
00146  * @v qword             Register buffer to read into
00147  * @v offset            Register offset
00148  */
00149 static void qib7322_readq ( struct qib7322 *qib7322, uint64_t *qword,
00150                             unsigned long offset ) {
00151         *qword = readq ( qib7322->regs + offset );
00152 }
00153 #define qib7322_readq( _qib7322, _ptr, _offset ) \
00154         qib7322_readq ( (_qib7322), (_ptr)->u.qwords, (_offset) )
00155 #define qib7322_readq_array8b( _qib7322, _ptr, _offset, _idx ) \
00156         qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) )
00157 #define qib7322_readq_array64k( _qib7322, _ptr, _offset, _idx ) \
00158         qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ) )
00159 #define qib7322_readq_port( _qib7322, _ptr, _offset, _port ) \
00160         qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) ) )
00161 
00162 /**
00163  * Write QIB7322 qword register
00164  *
00165  * @v qib7322           QIB7322 device
00166  * @v qword             Register buffer to write
00167  * @v offset            Register offset
00168  */
00169 static void qib7322_writeq ( struct qib7322 *qib7322, const uint64_t *qword,
00170                              unsigned long offset ) {
00171         writeq ( *qword, ( qib7322->regs + offset ) );
00172 }
00173 #define qib7322_writeq( _qib7322, _ptr, _offset ) \
00174         qib7322_writeq ( (_qib7322), (_ptr)->u.qwords, (_offset) )
00175 #define qib7322_writeq_array8b( _qib7322, _ptr, _offset, _idx ) \
00176         qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) )
00177 #define qib7322_writeq_array64k( _qib7322, _ptr, _offset, _idx ) \
00178         qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ))
00179 #define qib7322_writeq_port( _qib7322, _ptr, _offset, _port ) \
00180         qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) ))
00181 
00182 /**
00183  * Write QIB7322 dword register
00184  *
00185  * @v qib7322           QIB7322 device
00186  * @v dword             Value to write
00187  * @v offset            Register offset
00188  */
00189 static void qib7322_writel ( struct qib7322 *qib7322, uint32_t dword,
00190                              unsigned long offset ) {
00191         writel ( dword, ( qib7322->regs + offset ) );
00192 }
00193 
00194 /***************************************************************************
00195  *
00196  * Link state management
00197  *
00198  ***************************************************************************
00199  */
00200 
00201 /**
00202  * Textual representation of link state
00203  *
00204  * @v link_state        Link state
00205  * @ret link_text       Link state text
00206  */
00207 static const char * qib7322_link_state_text ( unsigned int link_state ) {
00208         switch ( link_state ) {
00209         case QIB7322_LINK_STATE_DOWN:           return "DOWN";
00210         case QIB7322_LINK_STATE_INIT:           return "INIT";
00211         case QIB7322_LINK_STATE_ARM:            return "ARM";
00212         case QIB7322_LINK_STATE_ACTIVE:         return "ACTIVE";
00213         case QIB7322_LINK_STATE_ACT_DEFER:      return "ACT_DEFER";
00214         default:                                return "UNKNOWN";
00215         }
00216 }
00217 
00218 /**
00219  * Handle link state change
00220  *
00221  * @v qib7322           QIB7322 device
00222  */
00223 static void qib7322_link_state_changed ( struct ib_device *ibdev ) {
00224         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00225         struct QIB_7322_IBCStatusA_0 ibcstatusa;
00226         struct QIB_7322_EXTCtrl extctrl;
00227         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00228         unsigned int link_training_state;
00229         unsigned int link_state;
00230         unsigned int link_width;
00231         unsigned int link_speed;
00232         unsigned int link_speed_qdr;
00233         unsigned int green;
00234         unsigned int yellow;
00235 
00236         /* Read link state */
00237         qib7322_readq_port ( qib7322, &ibcstatusa,
00238                              QIB_7322_IBCStatusA_0_offset, port );
00239         link_training_state = BIT_GET ( &ibcstatusa, LinkTrainingState );
00240         link_state = BIT_GET ( &ibcstatusa, LinkState );
00241         link_width = BIT_GET ( &ibcstatusa, LinkWidthActive );
00242         link_speed = BIT_GET ( &ibcstatusa, LinkSpeedActive );
00243         link_speed_qdr = BIT_GET ( &ibcstatusa, LinkSpeedQDR );
00244         DBGC ( qib7322, "QIB7322 %p port %d training state %#x link state %s "
00245                "(%s %s)\n", qib7322, port, link_training_state,
00246                qib7322_link_state_text ( link_state ),
00247                ( link_speed_qdr ? "QDR" : ( link_speed ? "DDR" : "SDR" ) ),
00248                ( link_width ? "x4" : "x1" ) );
00249 
00250         /* Set LEDs according to link state */
00251         qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
00252         green = ( ( link_state >= QIB7322_LINK_STATE_INIT ) ? 1 : 0 );
00253         yellow = ( ( link_state >= QIB7322_LINK_STATE_ACTIVE ) ? 1 : 0 );
00254         if ( port == 0 ) {
00255                 BIT_SET ( &extctrl, LEDPort0GreenOn, green );
00256                 BIT_SET ( &extctrl, LEDPort0YellowOn, yellow );
00257         } else {
00258                 BIT_SET ( &extctrl, LEDPort1GreenOn, green );
00259                 BIT_SET ( &extctrl, LEDPort1YellowOn, yellow );
00260         }
00261         qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
00262 
00263         /* Notify Infiniband core of link state change */
00264         ibdev->port_state = ( link_state + 1 );
00265         ibdev->link_width_active =
00266                 ( link_width ? IB_LINK_WIDTH_4X : IB_LINK_WIDTH_1X );
00267         ibdev->link_speed_active =
00268                 ( link_speed ? IB_LINK_SPEED_DDR : IB_LINK_SPEED_SDR );
00269         ib_link_state_changed ( ibdev );
00270 }
00271 
00272 /**
00273  * Wait for link state change to take effect
00274  *
00275  * @v ibdev             Infiniband device
00276  * @v new_link_state    Expected link state
00277  * @ret rc              Return status code
00278  */
00279 static int qib7322_link_state_check ( struct ib_device *ibdev,
00280                                       unsigned int new_link_state ) {
00281         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00282         struct QIB_7322_IBCStatusA_0 ibcstatusa;
00283         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00284         unsigned int link_state;
00285         unsigned int i;
00286 
00287         for ( i = 0 ; i < QIB7322_LINK_STATE_MAX_WAIT_US ; i++ ) {
00288                 qib7322_readq_port ( qib7322, &ibcstatusa,
00289                                      QIB_7322_IBCStatusA_0_offset, port );
00290                 link_state = BIT_GET ( &ibcstatusa, LinkState );
00291                 if ( link_state == new_link_state )
00292                         return 0;
00293                 udelay ( 1 );
00294         }
00295 
00296         DBGC ( qib7322, "QIB7322 %p port %d timed out waiting for link state "
00297                "%s\n", qib7322, port, qib7322_link_state_text ( link_state ) );
00298         return -ETIMEDOUT;
00299 }
00300 
00301 /**
00302  * Set port information
00303  *
00304  * @v ibdev             Infiniband device
00305  * @v mad               Set port information MAD
00306  */
00307 static int qib7322_set_port_info ( struct ib_device *ibdev,
00308                                    union ib_mad *mad ) {
00309         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00310         struct ib_port_info *port_info = &mad->smp.smp_data.port_info;
00311         struct QIB_7322_IBCCtrlA_0 ibcctrla;
00312         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00313         unsigned int port_state;
00314         unsigned int link_state;
00315 
00316         /* Set new link state */
00317         port_state = ( port_info->link_speed_supported__port_state & 0xf );
00318         if ( port_state ) {
00319                 link_state = ( port_state - 1 );
00320                 DBGC ( qib7322, "QIB7322 %p set link state to %s (%x)\n",
00321                        qib7322, qib7322_link_state_text ( link_state ),
00322                        link_state );
00323                 qib7322_readq_port ( qib7322, &ibcctrla,
00324                                      QIB_7322_IBCCtrlA_0_offset, port );
00325                 BIT_SET ( &ibcctrla, LinkCmd, link_state );
00326                 qib7322_writeq_port ( qib7322, &ibcctrla,
00327                                       QIB_7322_IBCCtrlA_0_offset, port );
00328 
00329                 /* Wait for link state change to take effect.  Ignore
00330                  * errors; the current link state will be returned via
00331                  * the GetResponse MAD.
00332                  */
00333                 qib7322_link_state_check ( ibdev, link_state );
00334         }
00335 
00336         /* Detect and report link state change */
00337         qib7322_link_state_changed ( ibdev );
00338 
00339         return 0;
00340 }
00341 
00342 /**
00343  * Set partition key table
00344  *
00345  * @v ibdev             Infiniband device
00346  * @v mad               Set partition key table MAD
00347  */
00348 static int qib7322_set_pkey_table ( struct ib_device *ibdev __unused,
00349                                     union ib_mad *mad __unused ) {
00350         /* Nothing to do */
00351         return 0;
00352 }
00353 
00354 /***************************************************************************
00355  *
00356  * Context allocation
00357  *
00358  ***************************************************************************
00359  */
00360 
00361 /**
00362  * Allocate a context and set queue pair number
00363  *
00364  * @v ibdev             Infiniband device
00365  * @v qp                Queue pair
00366  * @ret rc              Return status code
00367  */
00368 static int qib7322_alloc_ctx ( struct ib_device *ibdev,
00369                                struct ib_queue_pair *qp ) {
00370         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00371         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00372         unsigned int ctx;
00373 
00374         for ( ctx = port ; ctx < QIB7322_NUM_CONTEXTS ; ctx += 2 ) {
00375 
00376                 if ( ! qib7322->used_ctx[ctx] ) {
00377                         qib7322->used_ctx[ctx] = 1;
00378                         qp->qpn = ( ctx & ~0x01 );
00379                         DBGC2 ( qib7322, "QIB7322 %p port %d QPN %ld is CTX "
00380                                 "%d\n", qib7322, port, qp->qpn, ctx );
00381                         return 0;
00382                 }
00383         }
00384 
00385         DBGC ( qib7322, "QIB7322 %p port %d out of available contexts\n",
00386                qib7322, port );
00387         return -ENOENT;
00388 }
00389 
00390 /**
00391  * Get queue pair context number
00392  *
00393  * @v ibdev             Infiniband device
00394  * @v qp                Queue pair
00395  * @ret ctx             Context index
00396  */
00397 static unsigned int qib7322_ctx ( struct ib_device *ibdev,
00398                                   struct ib_queue_pair *qp ) {
00399         return ( qp->qpn + ( ibdev->port - QIB7322_PORT_BASE ) );
00400 }
00401 
00402 /**
00403  * Free a context
00404  *
00405  * @v qib7322           QIB7322 device
00406  * @v ctx               Context index
00407  */
00408 static void qib7322_free_ctx ( struct ib_device *ibdev,
00409                                struct ib_queue_pair *qp ) {
00410         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00411         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00412         unsigned int ctx = qib7322_ctx ( ibdev, qp );
00413 
00414         qib7322->used_ctx[ctx] = 0;
00415         DBGC2 ( qib7322, "QIB7322 %p port %d CTX %d freed\n",
00416                 qib7322, port, ctx );
00417 }
00418 
00419 /***************************************************************************
00420  *
00421  * Send datapath
00422  *
00423  ***************************************************************************
00424  */
00425 
00426 /** Send buffer toggle bit
00427  *
00428  * We encode send buffers as 15 bits of send buffer index plus a
00429  * single bit which should match the "check" bit in the SendBufAvail
00430  * array.
00431  */
00432 #define QIB7322_SEND_BUF_TOGGLE 0x8000
00433 
00434 /**
00435  * Create send buffer set
00436  *
00437  * @v qib7322           QIB7322 device
00438  * @v base              Send buffer base offset
00439  * @v size              Send buffer size
00440  * @v start             Index of first send buffer
00441  * @v count             Number of send buffers
00442  * @ret send_bufs       Send buffer set
00443  */
00444 static struct qib7322_send_buffers *
00445 qib7322_create_send_bufs ( struct qib7322 *qib7322, unsigned long base,
00446                            unsigned int size, unsigned int start,
00447                            unsigned int count ) {
00448         struct qib7322_send_buffers *send_bufs;
00449         unsigned int i;
00450 
00451         /* Allocate send buffer set */
00452         send_bufs = zalloc ( sizeof ( *send_bufs ) +
00453                              ( count * sizeof ( send_bufs->avail[0] ) ) );
00454         if ( ! send_bufs )
00455                 return NULL;
00456 
00457         /* Populate send buffer set */
00458         send_bufs->base = base;
00459         send_bufs->size = size;
00460         send_bufs->start = start;
00461         send_bufs->count = count;
00462         for ( i = 0 ; i < count ; i++ )
00463                 send_bufs->avail[i] = ( start + i );
00464 
00465         DBGC2 ( qib7322, "QIB7322 %p send buffer set %p [%d,%d] at %lx\n",
00466                 qib7322, send_bufs, start, ( start + count - 1 ),
00467                 send_bufs->base );
00468 
00469         return send_bufs;
00470 }
00471 
00472 /**
00473  * Destroy send buffer set
00474  *
00475  * @v qib7322           QIB7322 device
00476  * @v send_bufs         Send buffer set
00477  */
00478 static void
00479 qib7322_destroy_send_bufs ( struct qib7322 *qib7322 __unused,
00480                             struct qib7322_send_buffers *send_bufs ) {
00481         free ( send_bufs );
00482 }
00483 
00484 /**
00485  * Allocate a send buffer
00486  *
00487  * @v qib7322           QIB7322 device
00488  * @v send_bufs         Send buffer set
00489  * @ret send_buf        Send buffer, or negative error
00490  */
00491 static int qib7322_alloc_send_buf ( struct qib7322 *qib7322,
00492                                     struct qib7322_send_buffers *send_bufs ) {
00493         unsigned int used;
00494         unsigned int mask;
00495         unsigned int send_buf;
00496 
00497         used = ( send_bufs->cons - send_bufs->prod );
00498         if ( used >= send_bufs->count ) {
00499                 DBGC ( qib7322, "QIB7322 %p send buffer set %p out of "
00500                        "buffers\n", qib7322, send_bufs );
00501                 return -ENOBUFS;
00502         }
00503 
00504         mask = ( send_bufs->count - 1 );
00505         send_buf = send_bufs->avail[ send_bufs->cons++ & mask ];
00506         send_buf ^= QIB7322_SEND_BUF_TOGGLE;
00507         return send_buf;
00508 }
00509 
00510 /**
00511  * Free a send buffer
00512  *
00513  * @v qib7322           QIB7322 device
00514  * @v send_bufs         Send buffer set
00515  * @v send_buf          Send buffer
00516  */
00517 static void qib7322_free_send_buf ( struct qib7322 *qib7322 __unused,
00518                                     struct qib7322_send_buffers *send_bufs,
00519                                     unsigned int send_buf ) {
00520         unsigned int mask;
00521 
00522         mask = ( send_bufs->count - 1 );
00523         send_bufs->avail[ send_bufs->prod++ & mask ] = send_buf;
00524 }
00525 
00526 /**
00527  * Check to see if send buffer is in use
00528  *
00529  * @v qib7322           QIB7322 device
00530  * @v send_buf          Send buffer
00531  * @ret in_use          Send buffer is in use
00532  */
00533 static int qib7322_send_buf_in_use ( struct qib7322 *qib7322,
00534                                      unsigned int send_buf ) {
00535         unsigned int send_idx;
00536         unsigned int send_check;
00537         unsigned int inusecheck;
00538         unsigned int inuse;
00539         unsigned int check;
00540 
00541         send_idx = ( send_buf & ~QIB7322_SEND_BUF_TOGGLE );
00542         send_check = ( !! ( send_buf & QIB7322_SEND_BUF_TOGGLE ) );
00543         inusecheck = BIT_GET ( qib7322->sendbufavail, InUseCheck[send_idx] );
00544         inuse = ( !! ( inusecheck & 0x02 ) );
00545         check = ( !! ( inusecheck & 0x01 ) );
00546         return ( inuse || ( check != send_check ) );
00547 }
00548 
00549 /**
00550  * Calculate starting offset for send buffer
00551  *
00552  * @v qib7322           QIB7322 device
00553  * @v send_buf          Send buffer
00554  * @ret offset          Starting offset
00555  */
00556 static unsigned long
00557 qib7322_send_buffer_offset ( struct qib7322 *qib7322 __unused,
00558                              struct qib7322_send_buffers *send_bufs,
00559                              unsigned int send_buf ) {
00560         unsigned int index;
00561 
00562         index = ( ( send_buf & ~QIB7322_SEND_BUF_TOGGLE ) - send_bufs->start );
00563         return ( send_bufs->base + ( index * send_bufs->size ) );
00564 }
00565 
00566 /**
00567  * Create send work queue
00568  *
00569  * @v ibdev             Infiniband device
00570  * @v qp                Queue pair
00571  */
00572 static int qib7322_create_send_wq ( struct ib_device *ibdev,
00573                                     struct ib_queue_pair *qp ) {
00574         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00575         struct ib_work_queue *wq = &qp->send;
00576         struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
00577         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00578 
00579         /* Select send buffer set */
00580         if ( qp->type == IB_QPT_SMI ) {
00581                 if ( port == 0 ) {
00582                         qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port0;
00583                 } else {
00584                         qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port1;
00585                 }
00586         } else {
00587                 qib7322_wq->send_bufs = qib7322->send_bufs_small;
00588         }
00589 
00590         /* Allocate space for send buffer usage list */
00591         qib7322_wq->used = zalloc ( qp->send.num_wqes *
00592                                     sizeof ( qib7322_wq->used[0] ) );
00593         if ( ! qib7322_wq->used )
00594                 return -ENOMEM;
00595 
00596         /* Reset work queue */
00597         qib7322_wq->prod = 0;
00598         qib7322_wq->cons = 0;
00599 
00600         return 0;
00601 }
00602 
00603 /**
00604  * Destroy send work queue
00605  *
00606  * @v ibdev             Infiniband device
00607  * @v qp                Queue pair
00608  */
00609 static void qib7322_destroy_send_wq ( struct ib_device *ibdev __unused,
00610                                       struct ib_queue_pair *qp ) {
00611         struct ib_work_queue *wq = &qp->send;
00612         struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
00613 
00614         free ( qib7322_wq->used );
00615 }
00616 
00617 /**
00618  * Initialise send datapath
00619  *
00620  * @v qib7322           QIB7322 device
00621  * @ret rc              Return status code
00622  */
00623 static int qib7322_init_send ( struct qib7322 *qib7322 ) {
00624         struct QIB_7322_SendBufBase sendbufbase;
00625         struct QIB_7322_SendBufAvailAddr sendbufavailaddr;
00626         struct QIB_7322_SendCtrl sendctrl;
00627         struct QIB_7322_SendCtrl_0 sendctrlp;
00628         unsigned long baseaddr_smallpio;
00629         unsigned long baseaddr_largepio;
00630         unsigned long baseaddr_vl15_port0;
00631         unsigned long baseaddr_vl15_port1;
00632         int rc;
00633 
00634         /* Create send buffer sets */
00635         qib7322_readq ( qib7322, &sendbufbase, QIB_7322_SendBufBase_offset );
00636         baseaddr_smallpio = BIT_GET ( &sendbufbase, BaseAddr_SmallPIO );
00637         baseaddr_largepio = BIT_GET ( &sendbufbase, BaseAddr_LargePIO );
00638         baseaddr_vl15_port0 = ( baseaddr_largepio +
00639                                 ( QIB7322_LARGE_SEND_BUF_SIZE *
00640                                   QIB7322_LARGE_SEND_BUF_COUNT ) );
00641         baseaddr_vl15_port1 = ( baseaddr_vl15_port0 +
00642                                 QIB7322_VL15_PORT0_SEND_BUF_SIZE );
00643         qib7322->send_bufs_small =
00644                 qib7322_create_send_bufs ( qib7322, baseaddr_smallpio,
00645                                            QIB7322_SMALL_SEND_BUF_SIZE,
00646                                            QIB7322_SMALL_SEND_BUF_START,
00647                                            QIB7322_SMALL_SEND_BUF_USED );
00648         if ( ! qib7322->send_bufs_small ) {
00649                 rc = -ENOMEM;
00650                 goto err_create_send_bufs_small;
00651         }
00652         qib7322->send_bufs_vl15_port0 =
00653                 qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port0,
00654                                            QIB7322_VL15_PORT0_SEND_BUF_SIZE,
00655                                            QIB7322_VL15_PORT0_SEND_BUF_START,
00656                                            QIB7322_VL15_PORT0_SEND_BUF_COUNT );
00657         if ( ! qib7322->send_bufs_vl15_port0 ) {
00658                 rc = -ENOMEM;
00659                 goto err_create_send_bufs_vl15_port0;
00660         }
00661         qib7322->send_bufs_vl15_port1 =
00662                 qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port1,
00663                                            QIB7322_VL15_PORT1_SEND_BUF_SIZE,
00664                                            QIB7322_VL15_PORT1_SEND_BUF_START,
00665                                            QIB7322_VL15_PORT1_SEND_BUF_COUNT );
00666         if ( ! qib7322->send_bufs_vl15_port1 ) {
00667                 rc = -ENOMEM;
00668                 goto err_create_send_bufs_vl15_port1;
00669         }
00670 
00671         /* Allocate space for the SendBufAvail array */
00672         qib7322->sendbufavail = malloc_dma ( sizeof ( *qib7322->sendbufavail ),
00673                                              QIB7322_SENDBUFAVAIL_ALIGN );
00674         if ( ! qib7322->sendbufavail ) {
00675                 rc = -ENOMEM;
00676                 goto err_alloc_sendbufavail;
00677         }
00678         memset ( qib7322->sendbufavail, 0, sizeof ( *qib7322->sendbufavail ) );
00679 
00680         /* Program SendBufAvailAddr into the hardware */
00681         memset ( &sendbufavailaddr, 0, sizeof ( sendbufavailaddr ) );
00682         BIT_FILL_1 ( &sendbufavailaddr, SendBufAvailAddr,
00683                      ( virt_to_bus ( qib7322->sendbufavail ) >> 6 ) );
00684         qib7322_writeq ( qib7322, &sendbufavailaddr,
00685                          QIB_7322_SendBufAvailAddr_offset );
00686 
00687         /* Enable sending */
00688         memset ( &sendctrlp, 0, sizeof ( sendctrlp ) );
00689         BIT_FILL_1 ( &sendctrlp, SendEnable, 1 );
00690         qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_0_offset );
00691         qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_1_offset );
00692 
00693         /* Enable DMA of SendBufAvail */
00694         memset ( &sendctrl, 0, sizeof ( sendctrl ) );
00695         BIT_FILL_1 ( &sendctrl, SendBufAvailUpd, 1 );
00696         qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
00697 
00698         return 0;
00699 
00700         free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) );
00701  err_alloc_sendbufavail:
00702         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 );
00703  err_create_send_bufs_vl15_port1:
00704         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 );
00705  err_create_send_bufs_vl15_port0:
00706         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small );
00707  err_create_send_bufs_small:
00708         return rc;
00709 }
00710 
00711 /**
00712  * Shut down send datapath
00713  *
00714  * @v qib7322           QIB7322 device
00715  */
00716 static void qib7322_fini_send ( struct qib7322 *qib7322 ) {
00717         struct QIB_7322_SendCtrl sendctrl;
00718 
00719         /* Disable sending and DMA of SendBufAvail */
00720         memset ( &sendctrl, 0, sizeof ( sendctrl ) );
00721         qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
00722         mb();
00723 
00724         /* Ensure hardware has seen this disable */
00725         qib7322_readq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
00726 
00727         free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) );
00728         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 );
00729         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 );
00730         qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small );
00731 }
00732 
00733 /***************************************************************************
00734  *
00735  * Receive datapath
00736  *
00737  ***************************************************************************
00738  */
00739 
00740 /**
00741  * Create receive work queue
00742  *
00743  * @v ibdev             Infiniband device
00744  * @v qp                Queue pair
00745  * @ret rc              Return status code
00746  */
00747 static int qib7322_create_recv_wq ( struct ib_device *ibdev,
00748                                     struct ib_queue_pair *qp ) {
00749         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00750         struct ib_work_queue *wq = &qp->recv;
00751         struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
00752         struct QIB_7322_RcvHdrAddr0 rcvhdraddr;
00753         struct QIB_7322_RcvHdrTailAddr0 rcvhdrtailaddr;
00754         struct QIB_7322_RcvHdrHead0 rcvhdrhead;
00755         struct QIB_7322_scalar rcvegrindexhead;
00756         struct QIB_7322_RcvCtrl rcvctrl;
00757         struct QIB_7322_RcvCtrl_P rcvctrlp;
00758         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00759         unsigned int ctx = qib7322_ctx ( ibdev, qp );
00760         int rc;
00761 
00762         /* Reset context information */
00763         memset ( &qib7322_wq->header_prod, 0,
00764                  sizeof ( qib7322_wq->header_prod ) );
00765         qib7322_wq->header_cons = 0;
00766         qib7322_wq->eager_prod = 0;
00767         qib7322_wq->eager_cons = 0;
00768 
00769         /* Allocate receive header buffer */
00770         qib7322_wq->header = malloc_dma ( QIB7322_RECV_HEADERS_SIZE,
00771                                           QIB7322_RECV_HEADERS_ALIGN );
00772         if ( ! qib7322_wq->header ) {
00773                 rc = -ENOMEM;
00774                 goto err_alloc_header;
00775         }
00776 
00777         /* Enable context in hardware */
00778         memset ( &rcvhdraddr, 0, sizeof ( rcvhdraddr ) );
00779         BIT_FILL_1 ( &rcvhdraddr, RcvHdrAddr,
00780                      ( virt_to_bus ( qib7322_wq->header ) >> 2 ) );
00781         qib7322_writeq_array8b ( qib7322, &rcvhdraddr,
00782                                  QIB_7322_RcvHdrAddr0_offset, ctx );
00783         memset ( &rcvhdrtailaddr, 0, sizeof ( rcvhdrtailaddr ) );
00784         BIT_FILL_1 ( &rcvhdrtailaddr, RcvHdrTailAddr,
00785                      ( virt_to_bus ( &qib7322_wq->header_prod ) >> 2 ) );
00786         qib7322_writeq_array8b ( qib7322, &rcvhdrtailaddr,
00787                                  QIB_7322_RcvHdrTailAddr0_offset, ctx );
00788         memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) );
00789         BIT_FILL_1 ( &rcvhdrhead, counter, 1 );
00790         qib7322_writeq_array64k ( qib7322, &rcvhdrhead,
00791                                   QIB_7322_RcvHdrHead0_offset, ctx );
00792         memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) );
00793         BIT_FILL_1 ( &rcvegrindexhead, Value, 1 );
00794         qib7322_writeq_array64k ( qib7322, &rcvegrindexhead,
00795                                   QIB_7322_RcvEgrIndexHead0_offset, ctx );
00796         qib7322_readq_port ( qib7322, &rcvctrlp,
00797                              QIB_7322_RcvCtrl_0_offset, port );
00798         BIT_SET ( &rcvctrlp, ContextEnable[ctx], 1 );
00799         qib7322_writeq_port ( qib7322, &rcvctrlp,
00800                               QIB_7322_RcvCtrl_0_offset, port );
00801         qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00802         BIT_SET ( &rcvctrl, IntrAvail[ctx], 1 );
00803         qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00804 
00805         DBGC ( qib7322, "QIB7322 %p port %d QPN %ld CTX %d hdrs [%lx,%lx) prod "
00806                "%lx\n", qib7322, port, qp->qpn, ctx,
00807                virt_to_bus ( qib7322_wq->header ),
00808                ( virt_to_bus ( qib7322_wq->header )
00809                  + QIB7322_RECV_HEADERS_SIZE ),
00810                virt_to_bus ( &qib7322_wq->header_prod ) );
00811         return 0;
00812 
00813         free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE );
00814  err_alloc_header:
00815         return rc;
00816 }
00817 
00818 /**
00819  * Destroy receive work queue
00820  *
00821  * @v ibdev             Infiniband device
00822  * @v qp                Queue pair
00823  */
00824 static void qib7322_destroy_recv_wq ( struct ib_device *ibdev,
00825                                       struct ib_queue_pair *qp ) {
00826         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
00827         struct ib_work_queue *wq = &qp->recv;
00828         struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
00829         struct QIB_7322_RcvCtrl rcvctrl;
00830         struct QIB_7322_RcvCtrl_P rcvctrlp;
00831         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
00832         unsigned int ctx = qib7322_ctx ( ibdev, qp );
00833 
00834         /* Disable context in hardware */
00835         qib7322_readq_port ( qib7322, &rcvctrlp,
00836                              QIB_7322_RcvCtrl_0_offset, port );
00837         BIT_SET ( &rcvctrlp, ContextEnable[ctx], 0 );
00838         qib7322_writeq_port ( qib7322, &rcvctrlp,
00839                               QIB_7322_RcvCtrl_0_offset, port );
00840         qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00841         BIT_SET ( &rcvctrl, IntrAvail[ctx], 0 );
00842         qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00843 
00844         /* Make sure the hardware has seen that the context is disabled */
00845         qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00846         mb();
00847 
00848         /* Free headers ring */
00849         free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE );
00850 }
00851 
00852 /**
00853  * Initialise receive datapath
00854  *
00855  * @v qib7322           QIB7322 device
00856  * @ret rc              Return status code
00857  */
00858 static int qib7322_init_recv ( struct qib7322 *qib7322 ) {
00859         struct QIB_7322_RcvCtrl rcvctrl;
00860         struct QIB_7322_RcvCtrl_0 rcvctrlp;
00861         struct QIB_7322_RcvQPMapTableA_0 rcvqpmaptablea0;
00862         struct QIB_7322_RcvQPMapTableB_0 rcvqpmaptableb0;
00863         struct QIB_7322_RcvQPMapTableA_1 rcvqpmaptablea1;
00864         struct QIB_7322_RcvQPMapTableB_1 rcvqpmaptableb1;
00865         struct QIB_7322_RcvQPMulticastContext_0 rcvqpmcastctx0;
00866         struct QIB_7322_RcvQPMulticastContext_1 rcvqpmcastctx1;
00867         struct QIB_7322_scalar rcvegrbase;
00868         struct QIB_7322_scalar rcvhdrentsize;
00869         struct QIB_7322_scalar rcvhdrcnt;
00870         struct QIB_7322_RcvBTHQP_0 rcvbthqp;
00871         struct QIB_7322_RxCreditVL0_0 rxcreditvl;
00872         unsigned int contextcfg;
00873         unsigned long egrbase;
00874         unsigned int eager_array_size_kernel;
00875         unsigned int eager_array_size_user;
00876         unsigned int ctx;
00877 
00878         /* Select configuration based on number of contexts */
00879         switch ( QIB7322_NUM_CONTEXTS ) {
00880         case 6:
00881                 contextcfg = QIB7322_CONTEXTCFG_6CTX;
00882                 eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_6CTX_KERNEL;
00883                 eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_6CTX_USER;
00884                 break;
00885         case 10:
00886                 contextcfg = QIB7322_CONTEXTCFG_10CTX;
00887                 eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_10CTX_KERNEL;
00888                 eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_10CTX_USER;
00889                 break;
00890         case 18:
00891                 contextcfg = QIB7322_CONTEXTCFG_18CTX;
00892                 eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_18CTX_KERNEL;
00893                 eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_18CTX_USER;
00894                 break;
00895         default:
00896                 linker_assert ( 0, invalid_QIB7322_NUM_CONTEXTS );
00897                 return -EINVAL;
00898         }
00899 
00900         /* Configure number of contexts */
00901         memset ( &rcvctrl, 0, sizeof ( rcvctrl ) );
00902         BIT_FILL_2 ( &rcvctrl,
00903                      TailUpd, 1,
00904                      ContextCfg, contextcfg );
00905         qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
00906 
00907         /* Map QPNs to contexts */
00908         memset ( &rcvctrlp, 0, sizeof ( rcvctrlp ) );
00909         BIT_FILL_3 ( &rcvctrlp,
00910                      RcvIBPortEnable, 1,
00911                      RcvQPMapEnable, 1,
00912                      RcvPartitionKeyDisable, 1 );
00913         qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset );
00914         qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_1_offset );
00915         memset ( &rcvqpmaptablea0, 0, sizeof ( rcvqpmaptablea0 ) );
00916         BIT_FILL_6 ( &rcvqpmaptablea0,
00917                      RcvQPMapContext0, 0,
00918                      RcvQPMapContext1, 2,
00919                      RcvQPMapContext2, 4,
00920                      RcvQPMapContext3, 6,
00921                      RcvQPMapContext4, 8,
00922                      RcvQPMapContext5, 10 );
00923         qib7322_writeq ( qib7322, &rcvqpmaptablea0,
00924                          QIB_7322_RcvQPMapTableA_0_offset );
00925         memset ( &rcvqpmaptableb0, 0, sizeof ( rcvqpmaptableb0 ) );
00926         BIT_FILL_3 ( &rcvqpmaptableb0,
00927                      RcvQPMapContext6, 12,
00928                      RcvQPMapContext7, 14,
00929                      RcvQPMapContext8, 16 );
00930         qib7322_writeq ( qib7322, &rcvqpmaptableb0,
00931                          QIB_7322_RcvQPMapTableB_0_offset );
00932         memset ( &rcvqpmaptablea1, 0, sizeof ( rcvqpmaptablea1 ) );
00933         BIT_FILL_6 ( &rcvqpmaptablea1,
00934                      RcvQPMapContext0, 1,
00935                      RcvQPMapContext1, 3,
00936                      RcvQPMapContext2, 5,
00937                      RcvQPMapContext3, 7,
00938                      RcvQPMapContext4, 9,
00939                      RcvQPMapContext5, 11 );
00940         qib7322_writeq ( qib7322, &rcvqpmaptablea1,
00941                          QIB_7322_RcvQPMapTableA_1_offset );
00942         memset ( &rcvqpmaptableb1, 0, sizeof ( rcvqpmaptableb1 ) );
00943         BIT_FILL_3 ( &rcvqpmaptableb1,
00944                      RcvQPMapContext6, 13,
00945                      RcvQPMapContext7, 15,
00946                      RcvQPMapContext8, 17 );
00947         qib7322_writeq ( qib7322, &rcvqpmaptableb1,
00948                          QIB_7322_RcvQPMapTableB_1_offset );
00949 
00950         /* Map multicast QPNs to contexts */
00951         memset ( &rcvqpmcastctx0, 0, sizeof ( rcvqpmcastctx0 ) );
00952         BIT_FILL_1 ( &rcvqpmcastctx0, RcvQpMcContext, 0 );
00953         qib7322_writeq ( qib7322, &rcvqpmcastctx0,
00954                          QIB_7322_RcvQPMulticastContext_0_offset );
00955         memset ( &rcvqpmcastctx1, 0, sizeof ( rcvqpmcastctx1 ) );
00956         BIT_FILL_1 ( &rcvqpmcastctx1, RcvQpMcContext, 1 );
00957         qib7322_writeq ( qib7322, &rcvqpmcastctx1,
00958                          QIB_7322_RcvQPMulticastContext_1_offset );
00959 
00960         /* Configure receive header buffer sizes */
00961         memset ( &rcvhdrcnt, 0, sizeof ( rcvhdrcnt ) );
00962         BIT_FILL_1 ( &rcvhdrcnt, Value, QIB7322_RECV_HEADER_COUNT );
00963         qib7322_writeq ( qib7322, &rcvhdrcnt, QIB_7322_RcvHdrCnt_offset );
00964         memset ( &rcvhdrentsize, 0, sizeof ( rcvhdrentsize ) );
00965         BIT_FILL_1 ( &rcvhdrentsize, Value, ( QIB7322_RECV_HEADER_SIZE >> 2 ) );
00966         qib7322_writeq ( qib7322, &rcvhdrentsize,
00967                          QIB_7322_RcvHdrEntSize_offset );
00968 
00969         /* Calculate eager array start addresses for each context */
00970         qib7322_readq ( qib7322, &rcvegrbase, QIB_7322_RcvEgrBase_offset );
00971         egrbase = BIT_GET ( &rcvegrbase, Value );
00972         for ( ctx = 0 ; ctx < QIB7322_MAX_PORTS ; ctx++ ) {
00973                 qib7322->recv_wq[ctx].eager_array = egrbase;
00974                 qib7322->recv_wq[ctx].eager_entries = eager_array_size_kernel;
00975                 egrbase += ( eager_array_size_kernel *
00976                              sizeof ( struct QIB_7322_RcvEgr ) );
00977         }
00978         for ( ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) {
00979                 qib7322->recv_wq[ctx].eager_array = egrbase;
00980                 qib7322->recv_wq[ctx].eager_entries = eager_array_size_user;
00981                 egrbase += ( eager_array_size_user *
00982                              sizeof ( struct QIB_7322_RcvEgr ) );
00983         }
00984         for ( ctx = 0 ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) {
00985                 DBGC ( qib7322, "QIB7322 %p CTX %d eager array at %lx (%d "
00986                        "entries)\n", qib7322, ctx,
00987                        qib7322->recv_wq[ctx].eager_array,
00988                        qib7322->recv_wq[ctx].eager_entries );
00989         }
00990 
00991         /* Set the BTH QP for Infinipath packets to an unused value */
00992         memset ( &rcvbthqp, 0, sizeof ( rcvbthqp ) );
00993         BIT_FILL_1 ( &rcvbthqp, RcvBTHQP, QIB7322_QP_IDETH );
00994         qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_0_offset );
00995         qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_1_offset );
00996 
00997         /* Assign initial credits */
00998         memset ( &rxcreditvl, 0, sizeof ( rxcreditvl ) );
00999         BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL0 );
01000         qib7322_writeq_array8b ( qib7322, &rxcreditvl,
01001                                  QIB_7322_RxCreditVL0_0_offset, 0 );
01002         qib7322_writeq_array8b ( qib7322, &rxcreditvl,
01003                                  QIB_7322_RxCreditVL0_1_offset, 0 );
01004         BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL15 );
01005         qib7322_writeq_array8b ( qib7322, &rxcreditvl,
01006                                  QIB_7322_RxCreditVL0_0_offset, 15 );
01007         qib7322_writeq_array8b ( qib7322, &rxcreditvl,
01008                                  QIB_7322_RxCreditVL0_1_offset, 15 );
01009 
01010         return 0;
01011 }
01012 
01013 /**
01014  * Shut down receive datapath
01015  *
01016  * @v qib7322           QIB7322 device
01017  */
01018 static void qib7322_fini_recv ( struct qib7322 *qib7322 __unused ) {
01019         /* Nothing to do; all contexts were already disabled when the
01020          * queue pairs were destroyed
01021          */
01022 }
01023 
01024 /***************************************************************************
01025  *
01026  * Completion queue operations
01027  *
01028  ***************************************************************************
01029  */
01030 
01031 /**
01032  * Create completion queue
01033  *
01034  * @v ibdev             Infiniband device
01035  * @v cq                Completion queue
01036  * @ret rc              Return status code
01037  */
01038 static int qib7322_create_cq ( struct ib_device *ibdev,
01039                                struct ib_completion_queue *cq ) {
01040         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01041         static int cqn;
01042 
01043         /* The hardware has no concept of completion queues.  We
01044          * simply use the association between CQs and WQs (already
01045          * handled by the IB core) to decide which WQs to poll.
01046          *
01047          * We do set a CQN, just to avoid confusing debug messages
01048          * from the IB core.
01049          */
01050         cq->cqn = ++cqn;
01051         DBGC ( qib7322, "QIB7322 %p CQN %ld created\n", qib7322, cq->cqn );
01052 
01053         return 0;
01054 }
01055 
01056 /**
01057  * Destroy completion queue
01058  *
01059  * @v ibdev             Infiniband device
01060  * @v cq                Completion queue
01061  */
01062 static void qib7322_destroy_cq ( struct ib_device *ibdev,
01063                                  struct ib_completion_queue *cq ) {
01064         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01065 
01066         /* Nothing to do */
01067         DBGC ( qib7322, "QIB7322 %p CQN %ld destroyed\n", qib7322, cq->cqn );
01068 }
01069 
01070 /***************************************************************************
01071  *
01072  * Queue pair operations
01073  *
01074  ***************************************************************************
01075  */
01076 
01077 /**
01078  * Create queue pair
01079  *
01080  * @v ibdev             Infiniband device
01081  * @v qp                Queue pair
01082  * @ret rc              Return status code
01083  */
01084 static int qib7322_create_qp ( struct ib_device *ibdev,
01085                                struct ib_queue_pair *qp ) {
01086         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01087         unsigned int ctx;
01088         int rc;
01089 
01090         /* Allocate a context and QPN */
01091         if ( ( rc = qib7322_alloc_ctx ( ibdev, qp ) ) != 0 )
01092                 goto err_alloc_ctx;
01093         ctx = qib7322_ctx ( ibdev, qp );
01094 
01095         /* Set work-queue private data pointers */
01096         ib_wq_set_drvdata ( &qp->send, &qib7322->send_wq[ctx] );
01097         ib_wq_set_drvdata ( &qp->recv, &qib7322->recv_wq[ctx] );
01098 
01099         /* Create receive work queue */
01100         if ( ( rc = qib7322_create_recv_wq ( ibdev, qp ) ) != 0 )
01101                 goto err_create_recv_wq;
01102 
01103         /* Create send work queue */
01104         if ( ( rc = qib7322_create_send_wq ( ibdev, qp ) ) != 0 )
01105                 goto err_create_send_wq;
01106 
01107         return 0;
01108 
01109         qib7322_destroy_send_wq ( ibdev, qp );
01110  err_create_send_wq:
01111         qib7322_destroy_recv_wq ( ibdev, qp );
01112  err_create_recv_wq:
01113         qib7322_free_ctx ( ibdev, qp );
01114  err_alloc_ctx:
01115         return rc;
01116 }
01117 
01118 /**
01119  * Modify queue pair
01120  *
01121  * @v ibdev             Infiniband device
01122  * @v qp                Queue pair
01123  * @ret rc              Return status code
01124  */
01125 static int qib7322_modify_qp ( struct ib_device *ibdev,
01126                                struct ib_queue_pair *qp ) {
01127         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01128 
01129         /* Nothing to do; the hardware doesn't have a notion of queue
01130          * keys
01131          */
01132         DBGC2 ( qib7322, "QIB7322 %p QPN %ld modified\n", qib7322, qp->qpn );
01133         return 0;
01134 }
01135 
01136 /**
01137  * Destroy queue pair
01138  *
01139  * @v ibdev             Infiniband device
01140  * @v qp                Queue pair
01141  */
01142 static void qib7322_destroy_qp ( struct ib_device *ibdev,
01143                                  struct ib_queue_pair *qp ) {
01144 
01145         qib7322_destroy_send_wq ( ibdev, qp );
01146         qib7322_destroy_recv_wq ( ibdev, qp );
01147         qib7322_free_ctx ( ibdev, qp );
01148 }
01149 
01150 /***************************************************************************
01151  *
01152  * Work request operations
01153  *
01154  ***************************************************************************
01155  */
01156 
01157 /**
01158  * Post send work queue entry
01159  *
01160  * @v ibdev             Infiniband device
01161  * @v qp                Queue pair
01162  * @v dest              Destination address vector
01163  * @v iobuf             I/O buffer
01164  * @ret rc              Return status code
01165  */
01166 static int qib7322_post_send ( struct ib_device *ibdev,
01167                                struct ib_queue_pair *qp,
01168                                struct ib_address_vector *dest,
01169                                struct io_buffer *iobuf ) {
01170         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01171         struct ib_work_queue *wq = &qp->send;
01172         struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01173         struct QIB_7322_SendPbc sendpbc;
01174         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
01175         uint8_t header_buf[IB_MAX_HEADER_SIZE];
01176         struct io_buffer headers;
01177         int send_buf;
01178         unsigned long start_offset;
01179         unsigned long offset;
01180         size_t len;
01181         ssize_t frag_len;
01182         uint32_t *data;
01183 
01184         /* Allocate send buffer and calculate offset */
01185         send_buf = qib7322_alloc_send_buf ( qib7322, qib7322_wq->send_bufs );
01186         if ( send_buf < 0 )
01187                 return send_buf;
01188         start_offset = offset =
01189                 qib7322_send_buffer_offset ( qib7322, qib7322_wq->send_bufs,
01190                                              send_buf );
01191 
01192         /* Store I/O buffer and send buffer index */
01193         assert ( wq->iobufs[qib7322_wq->prod] == NULL );
01194         wq->iobufs[qib7322_wq->prod] = iobuf;
01195         qib7322_wq->used[qib7322_wq->prod] = send_buf;
01196 
01197         /* Construct headers */
01198         iob_populate ( &headers, header_buf, 0, sizeof ( header_buf ) );
01199         iob_reserve ( &headers, sizeof ( header_buf ) );
01200         ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
01201 
01202         /* Calculate packet length */
01203         len = ( ( sizeof ( sendpbc ) + iob_len ( &headers ) +
01204                   iob_len ( iobuf ) + 3 ) & ~3 );
01205 
01206         /* Construct send per-buffer control word */
01207         memset ( &sendpbc, 0, sizeof ( sendpbc ) );
01208         BIT_FILL_3 ( &sendpbc,
01209                      LengthP1_toibc, ( ( len >> 2 ) - 1 ),
01210                      Port, port,
01211                      VL15, ( ( qp->type == IB_QPT_SMI ) ? 1 : 0 ) );
01212 
01213         /* Write SendPbc */
01214         DBG_DISABLE ( DBGLVL_IO );
01215         qib7322_writeq ( qib7322, &sendpbc, offset );
01216         offset += sizeof ( sendpbc );
01217 
01218         /* Write headers */
01219         for ( data = headers.data, frag_len = iob_len ( &headers ) ;
01220               frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) {
01221                 qib7322_writel ( qib7322, *data, offset );
01222         }
01223 
01224         /* Write data */
01225         for ( data = iobuf->data, frag_len = iob_len ( iobuf ) ;
01226               frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) {
01227                 qib7322_writel ( qib7322, *data, offset );
01228         }
01229         DBG_ENABLE ( DBGLVL_IO );
01230 
01231         assert ( ( start_offset + len ) == offset );
01232         DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) posted [%lx,%lx)\n",
01233                 qib7322, qp->qpn, send_buf, qib7322_wq->prod,
01234                 start_offset, offset );
01235 
01236         /* Increment producer counter */
01237         qib7322_wq->prod = ( ( qib7322_wq->prod + 1 ) & ( wq->num_wqes - 1 ) );
01238 
01239         return 0;
01240 }
01241 
01242 /**
01243  * Complete send work queue entry
01244  *
01245  * @v ibdev             Infiniband device
01246  * @v qp                Queue pair
01247  * @v wqe_idx           Work queue entry index
01248  */
01249 static void qib7322_complete_send ( struct ib_device *ibdev,
01250                                     struct ib_queue_pair *qp,
01251                                     unsigned int wqe_idx ) {
01252         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01253         struct ib_work_queue *wq = &qp->send;
01254         struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01255         struct io_buffer *iobuf;
01256         unsigned int send_buf;
01257 
01258         /* Parse completion */
01259         send_buf = qib7322_wq->used[wqe_idx];
01260         DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) complete\n",
01261                 qib7322, qp->qpn, send_buf, wqe_idx );
01262 
01263         /* Complete work queue entry */
01264         iobuf = wq->iobufs[wqe_idx];
01265         assert ( iobuf != NULL );
01266         ib_complete_send ( ibdev, qp, iobuf, 0 );
01267         wq->iobufs[wqe_idx] = NULL;
01268 
01269         /* Free send buffer */
01270         qib7322_free_send_buf ( qib7322, qib7322_wq->send_bufs, send_buf );
01271 }
01272 
01273 /**
01274  * Poll send work queue
01275  *
01276  * @v ibdev             Infiniband device
01277  * @v qp                Queue pair
01278  */
01279 static void qib7322_poll_send_wq ( struct ib_device *ibdev,
01280                                    struct ib_queue_pair *qp ) {
01281         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01282         struct ib_work_queue *wq = &qp->send;
01283         struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01284         unsigned int send_buf;
01285 
01286         /* Look for completions */
01287         while ( wq->fill ) {
01288 
01289                 /* Check to see if send buffer has completed */
01290                 send_buf = qib7322_wq->used[qib7322_wq->cons];
01291                 if ( qib7322_send_buf_in_use ( qib7322, send_buf ) )
01292                         break;
01293 
01294                 /* Complete this buffer */
01295                 qib7322_complete_send ( ibdev, qp, qib7322_wq->cons );
01296 
01297                 /* Increment consumer counter */
01298                 qib7322_wq->cons = ( ( qib7322_wq->cons + 1 ) &
01299                                      ( wq->num_wqes - 1 ) );
01300         }
01301 }
01302 
01303 /**
01304  * Post receive work queue entry
01305  *
01306  * @v ibdev             Infiniband device
01307  * @v qp                Queue pair
01308  * @v iobuf             I/O buffer
01309  * @ret rc              Return status code
01310  */
01311 static int qib7322_post_recv ( struct ib_device *ibdev,
01312                                struct ib_queue_pair *qp,
01313                                struct io_buffer *iobuf ) {
01314         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01315         struct ib_work_queue *wq = &qp->recv;
01316         struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01317         struct QIB_7322_RcvEgr rcvegr;
01318         struct QIB_7322_scalar rcvegrindexhead;
01319         unsigned int ctx = qib7322_ctx ( ibdev, qp );
01320         physaddr_t addr;
01321         size_t len;
01322         unsigned int wqe_idx;
01323         unsigned int bufsize;
01324 
01325         /* Sanity checks */
01326         addr = virt_to_bus ( iobuf->data );
01327         len = iob_tailroom ( iobuf );
01328         if ( addr & ( QIB7322_EAGER_BUFFER_ALIGN - 1 ) ) {
01329                 DBGC ( qib7322, "QIB7322 %p QPN %ld misaligned RX buffer "
01330                        "(%08lx)\n", qib7322, qp->qpn, addr );
01331                 return -EINVAL;
01332         }
01333         if ( len != QIB7322_RECV_PAYLOAD_SIZE ) {
01334                 DBGC ( qib7322, "QIB7322 %p QPN %ld wrong RX buffer size "
01335                        "(%zd)\n", qib7322, qp->qpn, len );
01336                 return -EINVAL;
01337         }
01338 
01339         /* Calculate eager producer index and WQE index */
01340         wqe_idx = ( qib7322_wq->eager_prod & ( wq->num_wqes - 1 ) );
01341         assert ( wq->iobufs[wqe_idx] == NULL );
01342 
01343         /* Store I/O buffer */
01344         wq->iobufs[wqe_idx] = iobuf;
01345 
01346         /* Calculate buffer size */
01347         switch ( QIB7322_RECV_PAYLOAD_SIZE ) {
01348         case 2048:  bufsize = QIB7322_EAGER_BUFFER_2K;  break;
01349         case 4096:  bufsize = QIB7322_EAGER_BUFFER_4K;  break;
01350         case 8192:  bufsize = QIB7322_EAGER_BUFFER_8K;  break;
01351         case 16384: bufsize = QIB7322_EAGER_BUFFER_16K; break;
01352         case 32768: bufsize = QIB7322_EAGER_BUFFER_32K; break;
01353         case 65536: bufsize = QIB7322_EAGER_BUFFER_64K; break;
01354         default:    linker_assert ( 0, invalid_rx_payload_size );
01355                     bufsize = QIB7322_EAGER_BUFFER_NONE;
01356         }
01357 
01358         /* Post eager buffer */
01359         memset ( &rcvegr, 0, sizeof ( rcvegr ) );
01360         BIT_FILL_2 ( &rcvegr,
01361                      Addr, ( addr >> 11 ),
01362                      BufSize, bufsize );
01363         qib7322_writeq_array8b ( qib7322, &rcvegr, qib7322_wq->eager_array,
01364                                  qib7322_wq->eager_prod );
01365         DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x(%04x) posted "
01366                 "[%lx,%lx)\n", qib7322, qp->qpn, qib7322_wq->eager_prod,
01367                 wqe_idx, addr, ( addr + len ) );
01368 
01369         /* Increment producer index */
01370         qib7322_wq->eager_prod = ( ( qib7322_wq->eager_prod + 1 ) &
01371                                    ( qib7322_wq->eager_entries - 1 ) );
01372 
01373         /* Update head index */
01374         memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) );
01375         BIT_FILL_1 ( &rcvegrindexhead,
01376                      Value, ( ( qib7322_wq->eager_prod + 1 ) &
01377                               ( qib7322_wq->eager_entries - 1 ) ) );
01378         qib7322_writeq_array64k ( qib7322, &rcvegrindexhead,
01379                                   QIB_7322_RcvEgrIndexHead0_offset, ctx );
01380 
01381         return 0;
01382 }
01383 
01384 /**
01385  * Complete receive work queue entry
01386  *
01387  * @v ibdev             Infiniband device
01388  * @v qp                Queue pair
01389  * @v header_offs       Header offset
01390  */
01391 static void qib7322_complete_recv ( struct ib_device *ibdev,
01392                                     struct ib_queue_pair *qp,
01393                                     unsigned int header_offs ) {
01394         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01395         struct ib_work_queue *wq = &qp->recv;
01396         struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01397         struct QIB_7322_RcvHdrFlags *rcvhdrflags;
01398         struct QIB_7322_RcvEgr rcvegr;
01399         struct io_buffer headers;
01400         struct io_buffer *iobuf;
01401         struct ib_queue_pair *intended_qp;
01402         struct ib_address_vector dest;
01403         struct ib_address_vector source;
01404         unsigned int rcvtype;
01405         unsigned int pktlen;
01406         unsigned int egrindex;
01407         unsigned int useegrbfr;
01408         unsigned int iberr, mkerr, tiderr, khdrerr, mtuerr;
01409         unsigned int lenerr, parityerr, vcrcerr, icrcerr;
01410         unsigned int err;
01411         unsigned int hdrqoffset;
01412         unsigned int header_len;
01413         unsigned int padded_payload_len;
01414         unsigned int wqe_idx;
01415         size_t payload_len;
01416         int qp0;
01417         int rc;
01418 
01419         /* RcvHdrFlags are at the end of the header entry */
01420         rcvhdrflags = ( qib7322_wq->header + header_offs +
01421                         QIB7322_RECV_HEADER_SIZE - sizeof ( *rcvhdrflags ) );
01422         rcvtype = BIT_GET ( rcvhdrflags, RcvType );
01423         pktlen = ( BIT_GET ( rcvhdrflags, PktLen ) << 2 );
01424         egrindex = BIT_GET ( rcvhdrflags, EgrIndex );
01425         useegrbfr = BIT_GET ( rcvhdrflags, UseEgrBfr );
01426         hdrqoffset = ( BIT_GET ( rcvhdrflags, HdrqOffset ) << 2 );
01427         iberr = BIT_GET ( rcvhdrflags, IBErr );
01428         mkerr = BIT_GET ( rcvhdrflags, MKErr );
01429         tiderr = BIT_GET ( rcvhdrflags, TIDErr );
01430         khdrerr = BIT_GET ( rcvhdrflags, KHdrErr );
01431         mtuerr = BIT_GET ( rcvhdrflags, MTUErr );
01432         lenerr = BIT_GET ( rcvhdrflags, LenErr );
01433         parityerr = BIT_GET ( rcvhdrflags, ParityErr );
01434         vcrcerr = BIT_GET ( rcvhdrflags, VCRCErr );
01435         icrcerr = BIT_GET ( rcvhdrflags, ICRCErr );
01436         header_len = ( QIB7322_RECV_HEADER_SIZE - hdrqoffset -
01437                        sizeof ( *rcvhdrflags ) );
01438         padded_payload_len = ( pktlen - header_len - 4 /* ICRC */ );
01439         err = ( iberr | mkerr | tiderr | khdrerr | mtuerr |
01440                 lenerr | parityerr | vcrcerr | icrcerr );
01441         /* IB header is placed immediately before RcvHdrFlags */
01442         iob_populate ( &headers, ( ( ( void * ) rcvhdrflags ) - header_len ),
01443                        header_len, header_len );
01444 
01445         /* Dump diagnostic information */
01446         DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x%s hdr %d type %d len "
01447                 "%d(%d+%d+4)%s%s%s%s%s%s%s%s%s%s%s\n", qib7322, qp->qpn,
01448                 egrindex, ( useegrbfr ? "" : "(unused)" ),
01449                 ( header_offs / QIB7322_RECV_HEADER_SIZE ),
01450                 rcvtype, pktlen, header_len, padded_payload_len,
01451                 ( err ? " [Err" : "" ), ( iberr ? " IB" : "" ),
01452                 ( mkerr ? " MK" : "" ), ( tiderr ? " TID" : "" ),
01453                 ( khdrerr ? " KHdr" : "" ), ( mtuerr ? " MTU" : "" ),
01454                 ( lenerr ? " Len" : "" ), ( parityerr ? " Parity" : ""),
01455                 ( vcrcerr ? " VCRC" : "" ), ( icrcerr ? " ICRC" : "" ),
01456                 ( err ? "]" : "" ) );
01457         DBGCP_HDA ( qib7322, hdrqoffset, headers.data,
01458                     ( header_len + sizeof ( *rcvhdrflags ) ) );
01459 
01460         /* Parse header to generate address vector */
01461         qp0 = ( qp->qpn == 0 );
01462         intended_qp = NULL;
01463         if ( ( rc = ib_pull ( ibdev, &headers, ( qp0 ? &intended_qp : NULL ),
01464                               &payload_len, &dest, &source ) ) != 0 ) {
01465                 DBGC ( qib7322, "QIB7322 %p could not parse headers: %s\n",
01466                        qib7322, strerror ( rc ) );
01467                 err = 1;
01468         }
01469         if ( ! intended_qp )
01470                 intended_qp = qp;
01471 
01472         /* Complete this buffer and any skipped buffers.  Note that
01473          * when the hardware runs out of buffers, it will repeatedly
01474          * report the same buffer (the tail) as a TID error, and that
01475          * it also has a habit of sometimes skipping over several
01476          * buffers at once.
01477          */
01478         while ( 1 ) {
01479 
01480                 /* If we have caught up to the producer counter, stop.
01481                  * This will happen when the hardware first runs out
01482                  * of buffers and starts reporting TID errors against
01483                  * the eager buffer it wants to use next.
01484                  */
01485                 if ( qib7322_wq->eager_cons == qib7322_wq->eager_prod )
01486                         break;
01487 
01488                 /* If we have caught up to where we should be after
01489                  * completing this egrindex, stop.  We phrase the test
01490                  * this way to avoid completing the entire ring when
01491                  * we receive the same egrindex twice in a row.
01492                  */
01493                 if ( ( qib7322_wq->eager_cons ==
01494                        ( ( egrindex + 1 ) & ( qib7322_wq->eager_entries - 1 ))))
01495                         break;
01496 
01497                 /* Identify work queue entry and corresponding I/O
01498                  * buffer.
01499                  */
01500                 wqe_idx = ( qib7322_wq->eager_cons & ( wq->num_wqes - 1 ) );
01501                 iobuf = wq->iobufs[wqe_idx];
01502                 assert ( iobuf != NULL );
01503                 wq->iobufs[wqe_idx] = NULL;
01504 
01505                 /* Complete the eager buffer */
01506                 if ( qib7322_wq->eager_cons == egrindex ) {
01507                         /* Completing the eager buffer described in
01508                          * this header entry.
01509                          */
01510                         if ( payload_len <= iob_tailroom ( iobuf ) ) {
01511                                 iob_put ( iobuf, payload_len );
01512                                 rc = ( err ?
01513                                        -EIO : ( useegrbfr ? 0 : -ECANCELED ) );
01514                         } else {
01515                                 DBGC ( qib7322, "QIB7322 %p bad payload len "
01516                                        "%zd\n", qib7322, payload_len );
01517                                 rc = -EPROTO;
01518                         }
01519                         /* Redirect to target QP if necessary */
01520                         if ( qp != intended_qp ) {
01521                                 DBGC2 ( qib7322, "QIB7322 %p redirecting QPN "
01522                                         "%ld => %ld\n",
01523                                         qib7322, qp->qpn, intended_qp->qpn );
01524                                 /* Compensate for incorrect fill levels */
01525                                 qp->recv.fill--;
01526                                 intended_qp->recv.fill++;
01527                         }
01528                         ib_complete_recv ( ibdev, intended_qp, &dest, &source,
01529                                            iobuf, rc );
01530                 } else {
01531                         /* Completing on a skipped-over eager buffer */
01532                         ib_complete_recv ( ibdev, qp, &dest, &source, iobuf,
01533                                            -ECANCELED );
01534                 }
01535 
01536                 /* Clear eager buffer */
01537                 memset ( &rcvegr, 0, sizeof ( rcvegr ) );
01538                 qib7322_writeq_array8b ( qib7322, &rcvegr,
01539                                          qib7322_wq->eager_array,
01540                                          qib7322_wq->eager_cons );
01541 
01542                 /* Increment consumer index */
01543                 qib7322_wq->eager_cons = ( ( qib7322_wq->eager_cons + 1 ) &
01544                                            ( qib7322_wq->eager_entries - 1 ) );
01545         }
01546 }
01547 
01548 /**
01549  * Poll receive work queue
01550  *
01551  * @v ibdev             Infiniband device
01552  * @v qp                Queue pair
01553  */
01554 static void qib7322_poll_recv_wq ( struct ib_device *ibdev,
01555                                    struct ib_queue_pair *qp ) {
01556         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01557         struct ib_work_queue *wq = &qp->recv;
01558         struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
01559         struct QIB_7322_RcvHdrHead0 rcvhdrhead;
01560         unsigned int ctx = qib7322_ctx ( ibdev, qp );
01561         unsigned int header_prod;
01562 
01563         /* Check for received packets */
01564         header_prod = ( BIT_GET ( &qib7322_wq->header_prod, Value ) << 2 );
01565         if ( header_prod == qib7322_wq->header_cons )
01566                 return;
01567 
01568         /* Process all received packets */
01569         while ( qib7322_wq->header_cons != header_prod ) {
01570 
01571                 /* Complete the receive */
01572                 qib7322_complete_recv ( ibdev, qp, qib7322_wq->header_cons );
01573 
01574                 /* Increment the consumer offset */
01575                 qib7322_wq->header_cons += QIB7322_RECV_HEADER_SIZE;
01576                 qib7322_wq->header_cons %= QIB7322_RECV_HEADERS_SIZE;
01577 
01578                 /* QIB7322 has only one send buffer per port for VL15,
01579                  * which almost always leads to send buffer exhaustion
01580                  * and dropped MADs.  Mitigate this by refusing to
01581                  * process more than one VL15 MAD per poll, which will
01582                  * enforce interleaved TX/RX polls.
01583                  */
01584                 if ( qp->type == IB_QPT_SMI )
01585                         break;
01586         }
01587 
01588         /* Update consumer offset */
01589         memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) );
01590         BIT_FILL_2 ( &rcvhdrhead,
01591                      RcvHeadPointer, ( qib7322_wq->header_cons >> 2 ),
01592                      counter, 1 );
01593         qib7322_writeq_array64k ( qib7322, &rcvhdrhead,
01594                                   QIB_7322_RcvHdrHead0_offset, ctx );
01595 }
01596 
01597 /**
01598  * Poll completion queue
01599  *
01600  * @v ibdev             Infiniband device
01601  * @v cq                Completion queue
01602  */
01603 static void qib7322_poll_cq ( struct ib_device *ibdev,
01604                               struct ib_completion_queue *cq ) {
01605         struct ib_work_queue *wq;
01606 
01607         /* Poll associated send and receive queues */
01608         list_for_each_entry ( wq, &cq->work_queues, list ) {
01609                 if ( wq->is_send ) {
01610                         qib7322_poll_send_wq ( ibdev, wq->qp );
01611                 } else {
01612                         qib7322_poll_recv_wq ( ibdev, wq->qp );
01613                 }
01614         }
01615 }
01616 
01617 /***************************************************************************
01618  *
01619  * Event queues
01620  *
01621  ***************************************************************************
01622  */
01623 
01624 /**
01625  * Poll event queue
01626  *
01627  * @v ibdev             Infiniband device
01628  */
01629 static void qib7322_poll_eq ( struct ib_device *ibdev ) {
01630         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01631         struct QIB_7322_ErrStatus_0 errstatus;
01632         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
01633 
01634         /* Check for and clear status bits */
01635         DBG_DISABLE ( DBGLVL_IO );
01636         qib7322_readq_port ( qib7322, &errstatus,
01637                              QIB_7322_ErrStatus_0_offset, port );
01638         if ( errstatus.u.qwords[0] ) {
01639                 DBGC ( qib7322, "QIB7322 %p port %d status %08x%08x\n", qib7322,
01640                        port, errstatus.u.dwords[1],  errstatus.u.dwords[0] );
01641                 qib7322_writeq_port ( qib7322, &errstatus,
01642                                       QIB_7322_ErrClear_0_offset, port );
01643         }
01644         DBG_ENABLE ( DBGLVL_IO );
01645 
01646         /* Check for link status changes */
01647         if ( BIT_GET ( &errstatus, IBStatusChanged ) )
01648                 qib7322_link_state_changed ( ibdev );
01649 }
01650 
01651 /***************************************************************************
01652  *
01653  * Infiniband link-layer operations
01654  *
01655  ***************************************************************************
01656  */
01657 
01658 /**
01659  * Determine supported link speeds
01660  *
01661  * @v qib7322           QIB7322 device
01662  * @ret supported       Supported link speeds
01663  */
01664 static unsigned int qib7322_link_speed_supported ( struct qib7322 *qib7322,
01665                                                    unsigned int port ) {
01666         struct QIB_7322_feature_mask features;
01667         struct QIB_7322_Revision revision;
01668         unsigned int supported;
01669         unsigned int boardid;
01670 
01671         /* Read the active feature mask */
01672         qib7322_readq ( qib7322, &features,
01673                         QIB_7322_active_feature_mask_offset );
01674         switch ( port ) {
01675         case 0 :
01676                 supported = BIT_GET ( &features, Port0_Link_Speed_Supported );
01677                 break;
01678         case 1 :
01679                 supported = BIT_GET ( &features, Port1_Link_Speed_Supported );
01680                 break;
01681         default:
01682                 DBGC ( qib7322, "QIB7322 %p port %d is invalid\n",
01683                        qib7322, port );
01684                 supported = 0;
01685                 break;
01686         }
01687 
01688         /* Apply hacks for specific board IDs */
01689         qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset );
01690         boardid = BIT_GET ( &revision, BoardID );
01691         switch ( boardid ) {
01692         case QIB7322_BOARD_QMH7342 :
01693                 DBGC2 ( qib7322, "QIB7322 %p is a QMH7342; forcing QDR-only\n",
01694                         qib7322 );
01695                 supported = IB_LINK_SPEED_QDR;
01696                 break;
01697         default:
01698                 /* Do nothing */
01699                 break;
01700         }
01701 
01702         DBGC2 ( qib7322, "QIB7322 %p port %d %s%s%s%s\n", qib7322, port,
01703                 ( supported ? "supports" : "disabled" ),
01704                 ( ( supported & IB_LINK_SPEED_SDR ) ? " SDR" : "" ),
01705                 ( ( supported & IB_LINK_SPEED_DDR ) ? " DDR" : "" ),
01706                 ( ( supported & IB_LINK_SPEED_QDR ) ? " QDR" : "" ) );
01707         return supported;
01708 }
01709 
01710 /**
01711  * Initialise Infiniband link
01712  *
01713  * @v ibdev             Infiniband device
01714  * @ret rc              Return status code
01715  */
01716 static int qib7322_open ( struct ib_device *ibdev ) {
01717         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01718         struct QIB_7322_IBCCtrlA_0 ibcctrla;
01719         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
01720 
01721         /* Enable link */
01722         qib7322_readq_port ( qib7322, &ibcctrla,
01723                              QIB_7322_IBCCtrlA_0_offset, port );
01724         BIT_SET ( &ibcctrla, IBLinkEn, 1 );
01725         qib7322_writeq_port ( qib7322, &ibcctrla,
01726                               QIB_7322_IBCCtrlA_0_offset, port );
01727 
01728         return 0;
01729 }
01730 
01731 /**
01732  * Close Infiniband link
01733  *
01734  * @v ibdev             Infiniband device
01735  */
01736 static void qib7322_close ( struct ib_device *ibdev ) {
01737         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01738         struct QIB_7322_IBCCtrlA_0 ibcctrla;
01739         unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
01740 
01741         /* Disable link */
01742         qib7322_readq_port ( qib7322, &ibcctrla,
01743                              QIB_7322_IBCCtrlA_0_offset, port );
01744         BIT_SET ( &ibcctrla, IBLinkEn, 0 );
01745         qib7322_writeq_port ( qib7322, &ibcctrla,
01746                               QIB_7322_IBCCtrlA_0_offset, port );
01747 }
01748 
01749 /***************************************************************************
01750  *
01751  * Multicast group operations
01752  *
01753  ***************************************************************************
01754  */
01755 
01756 /**
01757  * Attach to multicast group
01758  *
01759  * @v ibdev             Infiniband device
01760  * @v qp                Queue pair
01761  * @v gid               Multicast GID
01762  * @ret rc              Return status code
01763  */
01764 static int qib7322_mcast_attach ( struct ib_device *ibdev,
01765                                   struct ib_queue_pair *qp,
01766                                   union ib_gid *gid ) {
01767         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01768 
01769         ( void ) qib7322;
01770         ( void ) qp;
01771         ( void ) gid;
01772         return 0;
01773 }
01774 
01775 /**
01776  * Detach from multicast group
01777  *
01778  * @v ibdev             Infiniband device
01779  * @v qp                Queue pair
01780  * @v gid               Multicast GID
01781  */
01782 static void qib7322_mcast_detach ( struct ib_device *ibdev,
01783                                    struct ib_queue_pair *qp,
01784                                    union ib_gid *gid ) {
01785         struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
01786 
01787         ( void ) qib7322;
01788         ( void ) qp;
01789         ( void ) gid;
01790  }
01791 
01792 /** QIB7322 Infiniband operations */
01793 static struct ib_device_operations qib7322_ib_operations = {
01794         .create_cq      = qib7322_create_cq,
01795         .destroy_cq     = qib7322_destroy_cq,
01796         .create_qp      = qib7322_create_qp,
01797         .modify_qp      = qib7322_modify_qp,
01798         .destroy_qp     = qib7322_destroy_qp,
01799         .post_send      = qib7322_post_send,
01800         .post_recv      = qib7322_post_recv,
01801         .poll_cq        = qib7322_poll_cq,
01802         .poll_eq        = qib7322_poll_eq,
01803         .open           = qib7322_open,
01804         .close          = qib7322_close,
01805         .mcast_attach   = qib7322_mcast_attach,
01806         .mcast_detach   = qib7322_mcast_detach,
01807         .set_port_info  = qib7322_set_port_info,
01808         .set_pkey_table = qib7322_set_pkey_table,
01809 };
01810 
01811 /***************************************************************************
01812  *
01813  * I2C bus operations
01814  *
01815  ***************************************************************************
01816  */
01817 
01818 /** QIB7322 I2C bit to GPIO mappings */
01819 static unsigned int qib7322_i2c_bits[] = {
01820         [I2C_BIT_SCL] = ( 1 << QIB7322_GPIO_SCL ),
01821         [I2C_BIT_SDA] = ( 1 << QIB7322_GPIO_SDA ),
01822 };
01823 
01824 /**
01825  * Read QIB7322 I2C line status
01826  *
01827  * @v basher            Bit-bashing interface
01828  * @v bit_id            Bit number
01829  * @ret zero            Input is a logic 0
01830  * @ret non-zero        Input is a logic 1
01831  */
01832 static int qib7322_i2c_read_bit ( struct bit_basher *basher,
01833                                   unsigned int bit_id ) {
01834         struct qib7322 *qib7322 =
01835                 container_of ( basher, struct qib7322, i2c.basher );
01836         struct QIB_7322_EXTStatus extstatus;
01837         unsigned int status;
01838 
01839         DBG_DISABLE ( DBGLVL_IO );
01840 
01841         qib7322_readq ( qib7322, &extstatus, QIB_7322_EXTStatus_offset );
01842         status = ( BIT_GET ( &extstatus, GPIOIn ) & qib7322_i2c_bits[bit_id] );
01843 
01844         DBG_ENABLE ( DBGLVL_IO );
01845 
01846         return status;
01847 }
01848 
01849 /**
01850  * Write QIB7322 I2C line status
01851  *
01852  * @v basher            Bit-bashing interface
01853  * @v bit_id            Bit number
01854  * @v data              Value to write
01855  */
01856 static void qib7322_i2c_write_bit ( struct bit_basher *basher,
01857                                     unsigned int bit_id, unsigned long data ) {
01858         struct qib7322 *qib7322 =
01859                 container_of ( basher, struct qib7322, i2c.basher );
01860         struct QIB_7322_EXTCtrl extctrl;
01861         struct QIB_7322_GPIO gpioout;
01862         unsigned int bit = qib7322_i2c_bits[bit_id];
01863         unsigned int outputs = 0;
01864         unsigned int output_enables = 0;
01865 
01866         DBG_DISABLE ( DBGLVL_IO );
01867 
01868         /* Read current GPIO mask and outputs */
01869         qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
01870         qib7322_readq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset );
01871 
01872         /* Update outputs and output enables.  I2C lines are tied
01873          * high, so we always set the output to 0 and use the output
01874          * enable to control the line.
01875          */
01876         output_enables = BIT_GET ( &extctrl, GPIOOe );
01877         output_enables = ( ( output_enables & ~bit ) | ( ~data & bit ) );
01878         outputs = BIT_GET ( &gpioout, GPIO );
01879         outputs = ( outputs & ~bit );
01880         BIT_SET ( &extctrl, GPIOOe, output_enables );
01881         BIT_SET ( &gpioout, GPIO, outputs );
01882 
01883         /* Write the output enable first; that way we avoid logic
01884          * hazards.
01885          */
01886         qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
01887         qib7322_writeq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset );
01888         mb();
01889 
01890         DBG_ENABLE ( DBGLVL_IO );
01891 }
01892 
01893 /** QIB7322 I2C bit-bashing interface operations */
01894 static struct bit_basher_operations qib7322_i2c_basher_ops = {
01895         .read   = qib7322_i2c_read_bit,
01896         .write  = qib7322_i2c_write_bit,
01897 };
01898 
01899 /**
01900  * Initialise QIB7322 I2C subsystem
01901  *
01902  * @v qib7322           QIB7322 device
01903  * @ret rc              Return status code
01904  */
01905 static int qib7322_init_i2c ( struct qib7322 *qib7322 ) {
01906         static int try_eeprom_address[] = { 0x51, 0x50 };
01907         unsigned int i;
01908         int rc;
01909 
01910         /* Initialise bus */
01911         if ( ( rc = init_i2c_bit_basher ( &qib7322->i2c,
01912                                           &qib7322_i2c_basher_ops ) ) != 0 ) {
01913                 DBGC ( qib7322, "QIB7322 %p could not initialise I2C bus: %s\n",
01914                        qib7322, strerror ( rc ) );
01915                 return rc;
01916         }
01917 
01918         /* Probe for devices */
01919         for ( i = 0 ; i < ( sizeof ( try_eeprom_address ) /
01920                             sizeof ( try_eeprom_address[0] ) ) ; i++ ) {
01921                 init_i2c_eeprom ( &qib7322->eeprom, try_eeprom_address[i] );
01922                 if ( ( rc = i2c_check_presence ( &qib7322->i2c.i2c,
01923                                                  &qib7322->eeprom ) ) == 0 ) {
01924                         DBGC2 ( qib7322, "QIB7322 %p found EEPROM at %02x\n",
01925                                 qib7322, try_eeprom_address[i] );
01926                         return 0;
01927                 }
01928         }
01929 
01930         DBGC ( qib7322, "QIB7322 %p could not find EEPROM\n", qib7322 );
01931         return -ENODEV;
01932 }
01933 
01934 /**
01935  * Read EEPROM parameters
01936  *
01937  * @v qib7322           QIB7322 device
01938  * @ret rc              Return status code
01939  */
01940 static int qib7322_read_eeprom ( struct qib7322 *qib7322 ) {
01941         struct i2c_interface *i2c = &qib7322->i2c.i2c;
01942         union ib_guid *guid = &qib7322->guid;
01943         int rc;
01944 
01945         /* Read GUID */
01946         if ( ( rc = i2c->read ( i2c, &qib7322->eeprom,
01947                                 QIB7322_EEPROM_GUID_OFFSET, guid->bytes,
01948                                 sizeof ( *guid ) ) ) != 0 ) {
01949                 DBGC ( qib7322, "QIB7322 %p could not read GUID: %s\n",
01950                        qib7322, strerror ( rc ) );
01951                 return rc;
01952         }
01953         DBGC2 ( qib7322, "QIB7322 %p has GUID " IB_GUID_FMT "\n",
01954                 qib7322, IB_GUID_ARGS ( guid ) );
01955 
01956         /* Read serial number (debug only) */
01957         if ( DBG_LOG ) {
01958                 uint8_t serial[QIB7322_EEPROM_SERIAL_SIZE + 1];
01959 
01960                 serial[ sizeof ( serial ) - 1 ] = '\0';
01961                 if ( ( rc = i2c->read ( i2c, &qib7322->eeprom,
01962                                         QIB7322_EEPROM_SERIAL_OFFSET, serial,
01963                                         ( sizeof ( serial ) - 1 ) ) ) != 0 ) {
01964                         DBGC ( qib7322, "QIB7322 %p could not read serial: "
01965                                "%s\n", qib7322, strerror ( rc ) );
01966                         return rc;
01967                 }
01968                 DBGC2 ( qib7322, "QIB7322 %p has serial number \"%s\"\n",
01969                         qib7322, serial );
01970         }
01971 
01972         return 0;
01973 }
01974 
01975 /***************************************************************************
01976  *
01977  * Advanced High-performance Bus (AHB) access
01978  *
01979  ***************************************************************************
01980  */
01981 
01982 /**
01983  * Wait for AHB transaction to complete
01984  *
01985  * @v qib7322           QIB7322 device
01986  * @ret rc              Return status code
01987  */
01988 static int qib7322_ahb_wait ( struct qib7322 *qib7322 ) {
01989         struct QIB_7322_ahb_transaction_reg transaction;
01990         unsigned int i;
01991 
01992         /* Wait for Ready bit to be asserted */
01993         for ( i = 0 ; i < QIB7322_AHB_MAX_WAIT_US ; i++ ) {
01994                 qib7322_readq ( qib7322, &transaction,
01995                                 QIB_7322_ahb_transaction_reg_offset );
01996                 if ( BIT_GET ( &transaction, ahb_rdy ) )
01997                         return 0;
01998                 udelay ( 1 );
01999         }
02000 
02001         DBGC ( qib7322, "QIB7322 %p timed out waiting for AHB transaction\n",
02002                qib7322 );
02003         return -ETIMEDOUT;
02004 }
02005 
02006 /**
02007  * Request ownership of the AHB
02008  *
02009  * @v qib7322           QIB7322 device
02010  * @v location          AHB location
02011  * @ret rc              Return status code
02012  */
02013 static int qib7322_ahb_request ( struct qib7322 *qib7322,
02014                                  unsigned int location ) {
02015         struct QIB_7322_ahb_access_ctrl access;
02016         int rc;
02017 
02018         /* Request ownership */
02019         memset ( &access, 0, sizeof ( access ) );
02020         BIT_FILL_2 ( &access,
02021                      sw_ahb_sel, 1,
02022                      sw_sel_ahb_trgt, QIB7322_AHB_LOC_TARGET ( location ) );
02023         qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset );
02024 
02025         /* Wait for ownership to be granted */
02026         if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )  {
02027                 DBGC ( qib7322, "QIB7322 %p could not obtain AHB ownership: "
02028                        "%s\n", qib7322, strerror ( rc ) );
02029                 return rc;
02030         }
02031 
02032         return 0;
02033 }
02034 
02035 /**
02036  * Release ownership of the AHB
02037  *
02038  * @v qib7322           QIB7322 device
02039  */
02040 static void qib7322_ahb_release ( struct qib7322 *qib7322 ) {
02041         struct QIB_7322_ahb_access_ctrl access;
02042 
02043         memset ( &access, 0, sizeof ( access ) );
02044         qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset );
02045 }
02046 
02047 /**
02048  * Read data via AHB
02049  *
02050  * @v qib7322           QIB7322 device
02051  * @v location          AHB location
02052  * @v data              Data to read
02053  * @ret rc              Return status code
02054  *
02055  * You must have already acquired ownership of the AHB.
02056  */
02057 static int qib7322_ahb_read ( struct qib7322 *qib7322, unsigned int location,
02058                               uint32_t *data ) {
02059         struct QIB_7322_ahb_transaction_reg xact;
02060         int rc;
02061 
02062         /* Avoid returning uninitialised data on error */
02063         *data = 0;
02064 
02065         /* Initiate transaction */
02066         memset ( &xact, 0, sizeof ( xact ) );
02067         BIT_FILL_2 ( &xact,
02068                      ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ),
02069                      write_not_read, 0 );
02070         qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
02071 
02072         /* Wait for transaction to complete */
02073         if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )
02074                 return rc;
02075 
02076         /* Read transaction data */
02077         qib7322_readq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
02078         *data = BIT_GET ( &xact, ahb_data );
02079         return 0;
02080 }
02081 
02082 /**
02083  * Write data via AHB
02084  *
02085  * @v qib7322           QIB7322 device
02086  * @v location          AHB location
02087  * @v data              Data to write
02088  * @ret rc              Return status code
02089  *
02090  * You must have already acquired ownership of the AHB.
02091  */
02092 static int qib7322_ahb_write ( struct qib7322 *qib7322, unsigned int location,
02093                                uint32_t data ) {
02094         struct QIB_7322_ahb_transaction_reg xact;
02095         int rc;
02096 
02097         /* Initiate transaction */
02098         memset ( &xact, 0, sizeof ( xact ) );
02099         BIT_FILL_3 ( &xact,
02100                      ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ),
02101                      write_not_read, 1,
02102                      ahb_data, data );
02103         qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
02104 
02105         /* Wait for transaction to complete */
02106         if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )
02107                 return rc;
02108 
02109         return 0;
02110 }
02111 
02112 /**
02113  * Read/modify/write AHB register
02114  *
02115  * @v qib7322           QIB7322 device
02116  * @v location          AHB location
02117  * @v value             Value to set
02118  * @v mask              Mask to apply to old value
02119  * @ret rc              Return status code
02120  */
02121 static int qib7322_ahb_mod_reg ( struct qib7322 *qib7322, unsigned int location,
02122                                  uint32_t value, uint32_t mask ) {
02123         uint32_t old_value;
02124         uint32_t new_value;
02125         int rc;
02126 
02127         DBG_DISABLE ( DBGLVL_IO );
02128 
02129         /* Sanity check */
02130         assert ( ( value & mask ) == value );
02131 
02132         /* Acquire bus ownership */
02133         if ( ( rc = qib7322_ahb_request ( qib7322, location ) ) != 0 )
02134                 goto out;
02135 
02136         /* Read existing value */
02137         if ( ( rc = qib7322_ahb_read ( qib7322, location, &old_value ) ) != 0 )
02138                 goto out_release;
02139 
02140         /* Update value */
02141         new_value = ( ( old_value & ~mask ) | value );
02142         DBGCP ( qib7322, "QIB7322 %p AHB %x %#08x => %#08x\n",
02143                 qib7322, location, old_value, new_value );
02144         if ( ( rc = qib7322_ahb_write ( qib7322, location, new_value ) ) != 0 )
02145                 goto out_release;
02146 
02147  out_release:
02148         /* Release bus */
02149         qib7322_ahb_release ( qib7322 );
02150  out:
02151         DBG_ENABLE ( DBGLVL_IO );
02152         return rc;
02153 }
02154 
02155 /**
02156  * Read/modify/write AHB register across all ports and channels
02157  *
02158  * @v qib7322           QIB7322 device
02159  * @v reg               AHB register
02160  * @v value             Value to set
02161  * @v mask              Mask to apply to old value
02162  * @ret rc              Return status code
02163  */
02164 static int qib7322_ahb_mod_reg_all ( struct qib7322 *qib7322, unsigned int reg,
02165                                      uint32_t value, uint32_t mask ) {
02166         unsigned int port;
02167         unsigned int channel;
02168         unsigned int location;
02169         int rc;
02170 
02171         for ( port = 0 ; port < QIB7322_MAX_PORTS ; port++ ) {
02172                 for ( channel = 0 ; channel < QIB7322_MAX_WIDTH ; channel++ ) {
02173                         location = QIB7322_AHB_LOCATION ( port, channel, reg );
02174                         if ( ( rc = qib7322_ahb_mod_reg ( qib7322, location,
02175                                                           value, mask ) ) != 0 )
02176                                 return rc;
02177                 }
02178         }
02179         return 0;
02180 }
02181 
02182 /***************************************************************************
02183  *
02184  * Infiniband SerDes initialisation
02185  *
02186  ***************************************************************************
02187  */
02188 
02189 /**
02190  * Initialise the IB SerDes
02191  *
02192  * @v qib7322           QIB7322 device
02193  * @ret rc              Return status code
02194  */
02195 static int qib7322_init_ib_serdes ( struct qib7322 *qib7322 ) {
02196         struct QIB_7322_IBCCtrlA_0 ibcctrla;
02197         struct QIB_7322_IBCCtrlB_0 ibcctrlb;
02198         struct QIB_7322_IBPCSConfig_0 ibpcsconfig;
02199 
02200         /* Configure sensible defaults for IBC */
02201         memset ( &ibcctrla, 0, sizeof ( ibcctrla ) );
02202         BIT_FILL_5 ( &ibcctrla, /* Tuning values taken from Linux driver */
02203                      FlowCtrlPeriod, 0x03,
02204                      FlowCtrlWaterMark, 0x05,
02205                      MaxPktLen, ( ( QIB7322_RECV_HEADER_SIZE +
02206                                     QIB7322_RECV_PAYLOAD_SIZE +
02207                                     4 /* ICRC */ ) >> 2 ),
02208                      PhyerrThreshold, 0xf,
02209                      OverrunThreshold, 0xf );
02210         qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset );
02211         qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_1_offset );
02212 
02213         /* Force SDR only to avoid needing all the DDR tuning,
02214          * Mellanox compatibility hacks etc.  SDR is plenty for
02215          * boot-time operation.
02216          */
02217         qib7322_readq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset );
02218         BIT_SET ( &ibcctrlb, IB_ENHANCED_MODE, 0 );
02219         BIT_SET ( &ibcctrlb, SD_SPEED_SDR, 1 );
02220         BIT_SET ( &ibcctrlb, SD_SPEED_DDR, 0 );
02221         BIT_SET ( &ibcctrlb, SD_SPEED_QDR, 0 );
02222         BIT_SET ( &ibcctrlb, IB_NUM_CHANNELS, 1 ); /* 4X only */
02223         BIT_SET ( &ibcctrlb, IB_LANE_REV_SUPPORTED, 0 );
02224         BIT_SET ( &ibcctrlb, HRTBT_ENB, 0 );
02225         BIT_SET ( &ibcctrlb, HRTBT_AUTO, 0 );
02226         qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset );
02227         qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_1_offset );
02228 
02229         /* Tune SerDes */
02230         qib7322_ahb_mod_reg_all ( qib7322, 2, 0, 0x00000e00UL );
02231 
02232         /* Bring XGXS out of reset */
02233         memset ( &ibpcsconfig, 0, sizeof ( ibpcsconfig ) );
02234         qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_0_offset );
02235         qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_1_offset );
02236 
02237         return 0;
02238 }
02239 
02240 /***************************************************************************
02241  *
02242  * PCI layer interface
02243  *
02244  ***************************************************************************
02245  */
02246 
02247 /**
02248  * Reset QIB7322
02249  *
02250  * @v qib7322           QIB7322 device
02251  * @v pci               PCI device
02252  * @ret rc              Return status code
02253  */
02254 static void qib7322_reset ( struct qib7322 *qib7322, struct pci_device *pci ) {
02255         struct QIB_7322_Control control;
02256         struct pci_config_backup backup;
02257 
02258         /* Back up PCI configuration space */
02259         pci_backup ( pci, &backup, NULL );
02260 
02261         /* Assert reset */
02262         memset ( &control, 0, sizeof ( control ) );
02263         BIT_FILL_1 ( &control, SyncReset, 1 );
02264         qib7322_writeq ( qib7322, &control, QIB_7322_Control_offset );
02265 
02266         /* Wait for reset to complete */
02267         mdelay ( 1000 );
02268 
02269         /* Restore PCI configuration space */
02270         pci_restore ( pci, &backup, NULL );
02271 }
02272 
02273 /**
02274  * Probe PCI device
02275  *
02276  * @v pci               PCI device
02277  * @v id                PCI ID
02278  * @ret rc              Return status code
02279  */
02280 static int qib7322_probe ( struct pci_device *pci ) {
02281         struct qib7322 *qib7322;
02282         struct QIB_7322_Revision revision;
02283         struct ib_device *ibdev;
02284         unsigned int link_speed_supported;
02285         int i;
02286         int rc;
02287 
02288         /* Allocate QIB7322 device */
02289         qib7322 = zalloc ( sizeof ( *qib7322 ) );
02290         if ( ! qib7322 ) {
02291                 rc = -ENOMEM;
02292                 goto err_alloc_qib7322;
02293         }
02294         pci_set_drvdata ( pci, qib7322 );
02295 
02296         /* Fix up PCI device */
02297         adjust_pci_device ( pci );
02298 
02299         /* Map PCI BARs */
02300         qib7322->regs = ioremap ( pci->membase, QIB7322_BAR0_SIZE );
02301         DBGC2 ( qib7322, "QIB7322 %p has BAR at %08lx\n",
02302                 qib7322, pci->membase );
02303 
02304         /* Reset device */
02305         qib7322_reset ( qib7322, pci );
02306 
02307         /* Print some general data */
02308         qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset );
02309         DBGC2 ( qib7322, "QIB7322 %p board %02lx v%ld.%ld.%ld.%ld\n", qib7322,
02310                 BIT_GET ( &revision, BoardID ),
02311                 BIT_GET ( &revision, R_SW ),
02312                 BIT_GET ( &revision, R_Arch ),
02313                 BIT_GET ( &revision, R_ChipRevMajor ),
02314                 BIT_GET ( &revision, R_ChipRevMinor ) );
02315 
02316         /* Initialise I2C subsystem */
02317         if ( ( rc = qib7322_init_i2c ( qib7322 ) ) != 0 )
02318                 goto err_init_i2c;
02319 
02320         /* Read EEPROM parameters */
02321         if ( ( rc = qib7322_read_eeprom ( qib7322 ) ) != 0 )
02322                 goto err_read_eeprom;
02323 
02324         /* Initialise send datapath */
02325         if ( ( rc = qib7322_init_send ( qib7322 ) ) != 0 )
02326                 goto err_init_send;
02327 
02328         /* Initialise receive datapath */
02329         if ( ( rc = qib7322_init_recv ( qib7322 ) ) != 0 )
02330                 goto err_init_recv;
02331 
02332         /* Initialise the IB SerDes */
02333         if ( ( rc = qib7322_init_ib_serdes ( qib7322 ) ) != 0 )
02334                 goto err_init_ib_serdes;
02335 
02336         /* Allocate Infiniband devices */
02337         for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) {
02338                 link_speed_supported =
02339                         qib7322_link_speed_supported ( qib7322, i );
02340                 if ( ! link_speed_supported )
02341                         continue;
02342                 ibdev = alloc_ibdev ( 0 );
02343                 if ( ! ibdev ) {
02344                         rc = -ENOMEM;
02345                         goto err_alloc_ibdev;
02346                 }
02347                 qib7322->ibdev[i] = ibdev;
02348                 ibdev->dev = &pci->dev;
02349                 ibdev->op = &qib7322_ib_operations;
02350                 ibdev->port = ( QIB7322_PORT_BASE + i );
02351                 ibdev->link_width_enabled = ibdev->link_width_supported =
02352                         IB_LINK_WIDTH_4X; /* 1x does not work */
02353                 ibdev->link_speed_enabled = ibdev->link_speed_supported =
02354                         IB_LINK_SPEED_SDR; /* to avoid need for link tuning */
02355                 memcpy ( &ibdev->node_guid, &qib7322->guid,
02356                          sizeof ( ibdev->node_guid ) );
02357                 memcpy ( &ibdev->gid.s.guid, &qib7322->guid,
02358                          sizeof ( ibdev->gid.s.guid ) );
02359                 assert ( ( ibdev->gid.s.guid.bytes[7] & i ) == 0 );
02360                 ibdev->gid.s.guid.bytes[7] |= i;
02361                 ib_set_drvdata ( ibdev, qib7322 );
02362         }
02363 
02364         /* Register Infiniband devices */
02365         for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) {
02366                 if ( ! qib7322->ibdev[i] )
02367                         continue;
02368                 if ( ( rc = register_ibdev ( qib7322->ibdev[i] ) ) != 0 ) {
02369                         DBGC ( qib7322, "QIB7322 %p port %d could not register "
02370                                "IB device: %s\n", qib7322, i, strerror ( rc ) );
02371                         goto err_register_ibdev;
02372                 }
02373         }
02374 
02375         return 0;
02376 
02377         i = QIB7322_MAX_PORTS;
02378  err_register_ibdev:
02379         for ( i-- ; i >= 0 ; i-- ) {
02380                 if ( qib7322->ibdev[i] )
02381                         unregister_ibdev ( qib7322->ibdev[i] );
02382         }
02383         i = QIB7322_MAX_PORTS;
02384  err_alloc_ibdev:
02385         for ( i-- ; i >= 0 ; i-- )
02386                 ibdev_put ( qib7322->ibdev[i] );
02387  err_init_ib_serdes:
02388         qib7322_fini_send ( qib7322 );
02389  err_init_send:
02390         qib7322_fini_recv ( qib7322 );
02391  err_init_recv:
02392  err_read_eeprom:
02393  err_init_i2c:
02394         iounmap ( qib7322->regs );
02395         free ( qib7322 );
02396  err_alloc_qib7322:
02397         return rc;
02398 }
02399 
02400 /**
02401  * Remove PCI device
02402  *
02403  * @v pci               PCI device
02404  */
02405 static void qib7322_remove ( struct pci_device *pci ) {
02406         struct qib7322 *qib7322 = pci_get_drvdata ( pci );
02407         int i;
02408 
02409         for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- ) {
02410                 if ( qib7322->ibdev[i] )
02411                         unregister_ibdev ( qib7322->ibdev[i] );
02412         }
02413         for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- )
02414                 ibdev_put ( qib7322->ibdev[i] );
02415         qib7322_fini_send ( qib7322 );
02416         qib7322_fini_recv ( qib7322 );
02417         iounmap ( qib7322->regs );
02418         free ( qib7322 );
02419 }
02420 
02421 static struct pci_device_id qib7322_nics[] = {
02422         PCI_ROM ( 0x1077, 0x7322, "iba7322", "IBA7322 QDR InfiniBand HCA", 0 ),
02423 };
02424 
02425 struct pci_driver qib7322_driver __pci_driver = {
02426         .ids = qib7322_nics,
02427         .id_count = ( sizeof ( qib7322_nics ) / sizeof ( qib7322_nics[0] ) ),
02428         .probe = qib7322_probe,
02429         .remove = qib7322_remove,
02430 };