iPXE
hermon.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
00003  * Copyright (C) 2008 Mellanox Technologies Ltd.
00004  *
00005  * This program is free software; you can redistribute it and/or
00006  * modify it under the terms of the GNU General Public License as
00007  * published by the Free Software Foundation; either version 2 of the
00008  * License, or any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful, but
00011  * WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013  * General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00018  * 02110-1301, USA.
00019  */
00020 
00021 FILE_LICENCE ( GPL2_OR_LATER );
00022 
00023 #include <stdint.h>
00024 #include <stdlib.h>
00025 #include <stdio.h>
00026 #include <string.h>
00027 #include <strings.h>
00028 #include <unistd.h>
00029 #include <errno.h>
00030 #include <byteswap.h>
00031 #include <ipxe/io.h>
00032 #include <ipxe/pci.h>
00033 #include <ipxe/pcibackup.h>
00034 #include <ipxe/malloc.h>
00035 #include <ipxe/umalloc.h>
00036 #include <ipxe/iobuf.h>
00037 #include <ipxe/netdevice.h>
00038 #include <ipxe/infiniband.h>
00039 #include <ipxe/ib_smc.h>
00040 #include <ipxe/if_ether.h>
00041 #include <ipxe/ethernet.h>
00042 #include <ipxe/fcoe.h>
00043 #include <ipxe/vlan.h>
00044 #include <ipxe/bofm.h>
00045 #include <ipxe/nvsvpd.h>
00046 #include <ipxe/nvo.h>
00047 #include "hermon.h"
00048 
00049 /**
00050  * @file
00051  *
00052  * Mellanox Hermon Infiniband HCA
00053  *
00054  */
00055 
00056 /***************************************************************************
00057  *
00058  * Queue number allocation
00059  *
00060  ***************************************************************************
00061  */
00062 
00063 /**
00064  * Allocate offsets within usage bitmask
00065  *
00066  * @v bits              Usage bitmask
00067  * @v bits_len          Length of usage bitmask
00068  * @v num_bits          Number of contiguous bits to allocate within bitmask
00069  * @ret bit             First free bit within bitmask, or negative error
00070  */
00071 static int hermon_bitmask_alloc ( hermon_bitmask_t *bits,
00072                                   unsigned int bits_len,
00073                                   unsigned int num_bits ) {
00074         unsigned int bit = 0;
00075         hermon_bitmask_t mask = 1;
00076         unsigned int found = 0;
00077 
00078         /* Search bits for num_bits contiguous free bits */
00079         while ( bit < bits_len ) {
00080                 if ( ( mask & *bits ) == 0 ) {
00081                         if ( ++found == num_bits )
00082                                 goto found;
00083                 } else {
00084                         found = 0;
00085                 }
00086                 bit++;
00087                 mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) );
00088                 if ( mask == 1 )
00089                         bits++;
00090         }
00091         return -ENFILE;
00092 
00093  found:
00094         /* Mark bits as in-use */
00095         do {
00096                 *bits |= mask;
00097                 if ( mask == 1 )
00098                         bits--;
00099                 mask = ( mask >> 1 ) | ( mask << ( 8 * sizeof ( mask ) - 1 ) );
00100         } while ( --found );
00101 
00102         return ( bit - num_bits + 1 );
00103 }
00104 
00105 /**
00106  * Free offsets within usage bitmask
00107  *
00108  * @v bits              Usage bitmask
00109  * @v bit               Starting bit within bitmask
00110  * @v num_bits          Number of contiguous bits to free within bitmask
00111  */
00112 static void hermon_bitmask_free ( hermon_bitmask_t *bits,
00113                                   int bit, unsigned int num_bits ) {
00114         hermon_bitmask_t mask;
00115 
00116         for ( ; num_bits ; bit++, num_bits-- ) {
00117                 mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) );
00118                 bits[ ( bit / ( 8 * sizeof ( mask ) ) ) ] &= ~mask;
00119         }
00120 }
00121 
00122 /***************************************************************************
00123  *
00124  * HCA commands
00125  *
00126  ***************************************************************************
00127  */
00128 
00129 /**
00130  * Wait for Hermon command completion
00131  *
00132  * @v hermon            Hermon device
00133  * @v hcr               HCA command registers
00134  * @ret rc              Return status code
00135  */
00136 static int hermon_cmd_wait ( struct hermon *hermon,
00137                              struct hermonprm_hca_command_register *hcr ) {
00138         unsigned int wait;
00139 
00140         for ( wait = HERMON_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
00141                 hcr->u.dwords[6] =
00142                         readl ( hermon->config + HERMON_HCR_REG ( 6 ) );
00143                 if ( ( MLX_GET ( hcr, go ) == 0 ) &&
00144                      ( MLX_GET ( hcr, t ) == hermon->toggle ) )
00145                         return 0;
00146                 mdelay ( 1 );
00147         }
00148         return -EBUSY;
00149 }
00150 
00151 /**
00152  * Issue HCA command
00153  *
00154  * @v hermon            Hermon device
00155  * @v command           Command opcode, flags and input/output lengths
00156  * @v op_mod            Opcode modifier (0 if no modifier applicable)
00157  * @v in                Input parameters
00158  * @v in_mod            Input modifier (0 if no modifier applicable)
00159  * @v out               Output parameters
00160  * @ret rc              Return status code
00161  */
00162 static int hermon_cmd ( struct hermon *hermon, unsigned long command,
00163                         unsigned int op_mod, const void *in,
00164                         unsigned int in_mod, void *out ) {
00165         struct hermonprm_hca_command_register hcr;
00166         unsigned int opcode = HERMON_HCR_OPCODE ( command );
00167         size_t in_len = HERMON_HCR_IN_LEN ( command );
00168         size_t out_len = HERMON_HCR_OUT_LEN ( command );
00169         void *in_buffer;
00170         void *out_buffer;
00171         unsigned int status;
00172         unsigned int i;
00173         int rc;
00174 
00175         assert ( in_len <= HERMON_MBOX_SIZE );
00176         assert ( out_len <= HERMON_MBOX_SIZE );
00177 
00178         DBGC2 ( hermon, "Hermon %p command %02x in %zx%s out %zx%s\n",
00179                 hermon, opcode, in_len,
00180                 ( ( command & HERMON_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
00181                 ( ( command & HERMON_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
00182 
00183         /* Check that HCR is free */
00184         if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
00185                 DBGC ( hermon, "Hermon %p command interface locked\n",
00186                        hermon );
00187                 return rc;
00188         }
00189 
00190         /* Flip HCR toggle */
00191         hermon->toggle = ( 1 - hermon->toggle );
00192 
00193         /* Prepare HCR */
00194         memset ( &hcr, 0, sizeof ( hcr ) );
00195         in_buffer = &hcr.u.dwords[0];
00196         if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
00197                 memset ( hermon->mailbox_in, 0, HERMON_MBOX_SIZE );
00198                 in_buffer = hermon->mailbox_in;
00199                 MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) );
00200                 MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
00201         }
00202         memcpy ( in_buffer, in, in_len );
00203         MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
00204         out_buffer = &hcr.u.dwords[3];
00205         if ( out_len && ( command & HERMON_HCR_OUT_MBOX ) ) {
00206                 out_buffer = hermon->mailbox_out;
00207                 MLX_FILL_H ( &hcr, 3, out_param_h,
00208                              virt_to_bus ( out_buffer ) );
00209                 MLX_FILL_1 ( &hcr, 4, out_param_l,
00210                              virt_to_bus ( out_buffer ) );
00211         }
00212         MLX_FILL_4 ( &hcr, 6,
00213                      opcode, opcode,
00214                      opcode_modifier, op_mod,
00215                      go, 1,
00216                      t, hermon->toggle );
00217         DBGC ( hermon, "Hermon %p issuing command %04x\n",
00218                hermon, opcode );
00219         DBGC2_HDA ( hermon, virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
00220                     &hcr, sizeof ( hcr ) );
00221         if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
00222                 DBGC2 ( hermon, "Input mailbox:\n" );
00223                 DBGC2_HDA ( hermon, virt_to_phys ( in_buffer ), in_buffer,
00224                             ( ( in_len < 512 ) ? in_len : 512 ) );
00225         }
00226 
00227         /* Issue command */
00228         for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
00229               i++ ) {
00230                 writel ( hcr.u.dwords[i],
00231                          hermon->config + HERMON_HCR_REG ( i ) );
00232                 barrier();
00233         }
00234 
00235         /* Wait for command completion */
00236         if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
00237                 DBGC ( hermon, "Hermon %p timed out waiting for command:\n",
00238                        hermon );
00239                 DBGC_HDA ( hermon,
00240                            virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
00241                            &hcr, sizeof ( hcr ) );
00242                 return rc;
00243         }
00244 
00245         /* Check command status */
00246         status = MLX_GET ( &hcr, status );
00247         if ( status != 0 ) {
00248                 DBGC ( hermon, "Hermon %p command failed with status %02x:\n",
00249                        hermon, status );
00250                 DBGC_HDA ( hermon,
00251                            virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
00252                            &hcr, sizeof ( hcr ) );
00253                 return -EIO;
00254         }
00255 
00256         /* Read output parameters, if any */
00257         hcr.u.dwords[3] = readl ( hermon->config + HERMON_HCR_REG ( 3 ) );
00258         hcr.u.dwords[4] = readl ( hermon->config + HERMON_HCR_REG ( 4 ) );
00259         memcpy ( out, out_buffer, out_len );
00260         if ( out_len ) {
00261                 DBGC2 ( hermon, "Output%s:\n",
00262                         ( command & HERMON_HCR_OUT_MBOX ) ? " mailbox" : "" );
00263                 DBGC2_HDA ( hermon, virt_to_phys ( out_buffer ), out_buffer,
00264                             ( ( out_len < 512 ) ? out_len : 512 ) );
00265         }
00266 
00267         return 0;
00268 }
00269 
00270 static inline int
00271 hermon_cmd_query_dev_cap ( struct hermon *hermon,
00272                            struct hermonprm_query_dev_cap *dev_cap ) {
00273         return hermon_cmd ( hermon,
00274                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_DEV_CAP,
00275                                                  1, sizeof ( *dev_cap ) ),
00276                             0, NULL, 0, dev_cap );
00277 }
00278 
00279 static inline int
00280 hermon_cmd_query_fw ( struct hermon *hermon, struct hermonprm_query_fw *fw ) {
00281         return hermon_cmd ( hermon,
00282                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_FW,
00283                                                  1, sizeof ( *fw ) ),
00284                             0, NULL, 0, fw );
00285 }
00286 
00287 static inline int
00288 hermon_cmd_init_hca ( struct hermon *hermon,
00289                       const struct hermonprm_init_hca *init_hca ) {
00290         return hermon_cmd ( hermon,
00291                             HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_HCA,
00292                                                 1, sizeof ( *init_hca ) ),
00293                             0, init_hca, 0, NULL );
00294 }
00295 
00296 static inline int
00297 hermon_cmd_close_hca ( struct hermon *hermon ) {
00298         return hermon_cmd ( hermon,
00299                             HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_HCA ),
00300                             0, NULL, 0, NULL );
00301 }
00302 
00303 static inline int
00304 hermon_cmd_init_port ( struct hermon *hermon, unsigned int port ) {
00305         return hermon_cmd ( hermon,
00306                             HERMON_HCR_VOID_CMD ( HERMON_HCR_INIT_PORT ),
00307                             0, NULL, port, NULL );
00308 }
00309 
00310 static inline int
00311 hermon_cmd_close_port ( struct hermon *hermon, unsigned int port ) {
00312         return hermon_cmd ( hermon,
00313                             HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_PORT ),
00314                             0, NULL, port, NULL );
00315 }
00316 
00317 static inline int
00318 hermon_cmd_set_port ( struct hermon *hermon, int is_ethernet,
00319                       unsigned int port_selector,
00320                       const union hermonprm_set_port *set_port ) {
00321         return hermon_cmd ( hermon,
00322                             HERMON_HCR_IN_CMD ( HERMON_HCR_SET_PORT,
00323                                                 1, sizeof ( *set_port ) ),
00324                             is_ethernet, set_port, port_selector, NULL );
00325 }
00326 
00327 static inline int
00328 hermon_cmd_sw2hw_mpt ( struct hermon *hermon, unsigned int index,
00329                        const struct hermonprm_mpt *mpt ) {
00330         return hermon_cmd ( hermon,
00331                             HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_MPT,
00332                                                 1, sizeof ( *mpt ) ),
00333                             0, mpt, index, NULL );
00334 }
00335 
00336 static inline int
00337 hermon_cmd_write_mtt ( struct hermon *hermon,
00338                        const struct hermonprm_write_mtt *write_mtt ) {
00339         return hermon_cmd ( hermon,
00340                             HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MTT,
00341                                                 1, sizeof ( *write_mtt ) ),
00342                             0, write_mtt, 1, NULL );
00343 }
00344 
00345 static inline int
00346 hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map,
00347                     const struct hermonprm_event_mask *mask ) {
00348         return hermon_cmd ( hermon,
00349                             HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ,
00350                                                 0, sizeof ( *mask ) ),
00351                             0, mask, index_map, NULL );
00352 }
00353 
00354 static inline int
00355 hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index,
00356                       const struct hermonprm_eqc *eqctx ) {
00357         return hermon_cmd ( hermon,
00358                             HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ,
00359                                                 1, sizeof ( *eqctx ) ),
00360                             0, eqctx, index, NULL );
00361 }
00362 
00363 static inline int
00364 hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index,
00365                       struct hermonprm_eqc *eqctx ) {
00366         return hermon_cmd ( hermon,
00367                             HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ,
00368                                                  1, sizeof ( *eqctx ) ),
00369                             1, NULL, index, eqctx );
00370 }
00371 
00372 static inline int
00373 hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index,
00374                       struct hermonprm_eqc *eqctx ) {
00375         return hermon_cmd ( hermon,
00376                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ,
00377                                                  1, sizeof ( *eqctx ) ),
00378                             0, NULL, index, eqctx );
00379 }
00380 
00381 static inline int
00382 hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn,
00383                       const struct hermonprm_completion_queue_context *cqctx ){
00384         return hermon_cmd ( hermon,
00385                             HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_CQ,
00386                                                 1, sizeof ( *cqctx ) ),
00387                             0, cqctx, cqn, NULL );
00388 }
00389 
00390 static inline int
00391 hermon_cmd_hw2sw_cq ( struct hermon *hermon, unsigned long cqn,
00392                       struct hermonprm_completion_queue_context *cqctx ) {
00393         return hermon_cmd ( hermon,
00394                             HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_CQ,
00395                                                  1, sizeof ( *cqctx ) ),
00396                             0, NULL, cqn, cqctx );
00397 }
00398 
00399 static inline int
00400 hermon_cmd_query_cq ( struct hermon *hermon, unsigned long cqn,
00401                       struct hermonprm_completion_queue_context *cqctx ) {
00402         return hermon_cmd ( hermon,
00403                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_CQ,
00404                                                  1, sizeof ( *cqctx ) ),
00405                             0, NULL, cqn, cqctx );
00406 }
00407 
00408 static inline int
00409 hermon_cmd_rst2init_qp ( struct hermon *hermon, unsigned long qpn,
00410                          const struct hermonprm_qp_ee_state_transitions *ctx ){
00411         return hermon_cmd ( hermon,
00412                             HERMON_HCR_IN_CMD ( HERMON_HCR_RST2INIT_QP,
00413                                                 1, sizeof ( *ctx ) ),
00414                             0, ctx, qpn, NULL );
00415 }
00416 
00417 static inline int
00418 hermon_cmd_init2rtr_qp ( struct hermon *hermon, unsigned long qpn,
00419                          const struct hermonprm_qp_ee_state_transitions *ctx ){
00420         return hermon_cmd ( hermon,
00421                             HERMON_HCR_IN_CMD ( HERMON_HCR_INIT2RTR_QP,
00422                                                 1, sizeof ( *ctx ) ),
00423                             0, ctx, qpn, NULL );
00424 }
00425 
00426 static inline int
00427 hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn,
00428                         const struct hermonprm_qp_ee_state_transitions *ctx ) {
00429         return hermon_cmd ( hermon,
00430                             HERMON_HCR_IN_CMD ( HERMON_HCR_RTR2RTS_QP,
00431                                                 1, sizeof ( *ctx ) ),
00432                             0, ctx, qpn, NULL );
00433 }
00434 
00435 static inline int
00436 hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn,
00437                         const struct hermonprm_qp_ee_state_transitions *ctx ) {
00438         return hermon_cmd ( hermon,
00439                             HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP,
00440                                                 1, sizeof ( *ctx ) ),
00441                             0, ctx, qpn, NULL );
00442 }
00443 
00444 static inline int
00445 hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) {
00446         return hermon_cmd ( hermon,
00447                             HERMON_HCR_VOID_CMD ( HERMON_HCR_2RST_QP ),
00448                             0x03, NULL, qpn, NULL );
00449 }
00450 
00451 static inline int
00452 hermon_cmd_query_qp ( struct hermon *hermon, unsigned long qpn,
00453                       struct hermonprm_qp_ee_state_transitions *ctx ) {
00454         return hermon_cmd ( hermon,
00455                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_QP,
00456                                                  1, sizeof ( *ctx ) ),
00457                             0, NULL, qpn, ctx );
00458 }
00459 
00460 static inline int
00461 hermon_cmd_conf_special_qp ( struct hermon *hermon, unsigned int internal_qps,
00462                              unsigned long base_qpn ) {
00463         return hermon_cmd ( hermon,
00464                             HERMON_HCR_VOID_CMD ( HERMON_HCR_CONF_SPECIAL_QP ),
00465                             internal_qps, NULL, base_qpn, NULL );
00466 }
00467 
00468 static inline int
00469 hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port,
00470                      union hermonprm_mad *mad ) {
00471         return hermon_cmd ( hermon,
00472                             HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC,
00473                                                    1, sizeof ( *mad ),
00474                                                    1, sizeof ( *mad ) ),
00475                             0x03, mad, port, mad );
00476 }
00477 
00478 static inline int
00479 hermon_cmd_read_mcg ( struct hermon *hermon, unsigned int index,
00480                       struct hermonprm_mcg_entry *mcg ) {
00481         return hermon_cmd ( hermon,
00482                             HERMON_HCR_OUT_CMD ( HERMON_HCR_READ_MCG,
00483                                                  1, sizeof ( *mcg ) ),
00484                             0, NULL, index, mcg );
00485 }
00486 
00487 static inline int
00488 hermon_cmd_write_mcg ( struct hermon *hermon, unsigned int index,
00489                        const struct hermonprm_mcg_entry *mcg ) {
00490         return hermon_cmd ( hermon,
00491                             HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MCG,
00492                                                 1, sizeof ( *mcg ) ),
00493                             0, mcg, index, NULL );
00494 }
00495 
00496 static inline int
00497 hermon_cmd_mgid_hash ( struct hermon *hermon, const union ib_gid *gid,
00498                        struct hermonprm_mgm_hash *hash ) {
00499         return hermon_cmd ( hermon,
00500                             HERMON_HCR_INOUT_CMD ( HERMON_HCR_MGID_HASH,
00501                                                    1, sizeof ( *gid ),
00502                                                    0, sizeof ( *hash ) ),
00503                             0, gid, 0, hash );
00504 }
00505 
00506 static inline int
00507 hermon_cmd_mod_stat_cfg ( struct hermon *hermon, unsigned int mode,
00508                           unsigned int input_mod,
00509                           struct hermonprm_scalar_parameter *portion ) {
00510         return hermon_cmd ( hermon,
00511                             HERMON_HCR_INOUT_CMD ( HERMON_HCR_MOD_STAT_CFG,
00512                                                    0, sizeof ( *portion ),
00513                                                    0, sizeof ( *portion ) ),
00514                             mode, portion, input_mod, portion );
00515 }
00516 
00517 static inline int
00518 hermon_cmd_query_port ( struct hermon *hermon, unsigned int port,
00519                         struct hermonprm_query_port_cap *query_port ) {
00520         return hermon_cmd ( hermon,
00521                             HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_PORT,
00522                                                  1, sizeof ( *query_port ) ),
00523                             0, NULL, port, query_port );
00524 }
00525 
00526 static inline int
00527 hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
00528                         struct hermonprm_sense_port *port_type ) {
00529         return hermon_cmd ( hermon,
00530                             HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
00531                                                  0, sizeof ( *port_type ) ),
00532                             0, NULL, port, port_type );
00533 }
00534 
00535 static inline int
00536 hermon_cmd_run_fw ( struct hermon *hermon ) {
00537         return hermon_cmd ( hermon,
00538                             HERMON_HCR_VOID_CMD ( HERMON_HCR_RUN_FW ),
00539                             0, NULL, 0, NULL );
00540 }
00541 
00542 static inline int
00543 hermon_cmd_unmap_icm ( struct hermon *hermon, unsigned int page_count,
00544                        const struct hermonprm_scalar_parameter *offset ) {
00545         return hermon_cmd ( hermon,
00546                             HERMON_HCR_IN_CMD ( HERMON_HCR_UNMAP_ICM,
00547                                                 0, sizeof ( *offset ) ),
00548                             0, offset, page_count, NULL );
00549 }
00550 
00551 static inline int
00552 hermon_cmd_map_icm ( struct hermon *hermon,
00553                      const struct hermonprm_virtual_physical_mapping *map ) {
00554         return hermon_cmd ( hermon,
00555                             HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM,
00556                                                 1, sizeof ( *map ) ),
00557                             0, map, 1, NULL );
00558 }
00559 
00560 static inline int
00561 hermon_cmd_unmap_icm_aux ( struct hermon *hermon ) {
00562         return hermon_cmd ( hermon,
00563                             HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_ICM_AUX ),
00564                             0, NULL, 0, NULL );
00565 }
00566 
00567 static inline int
00568 hermon_cmd_map_icm_aux ( struct hermon *hermon,
00569                        const struct hermonprm_virtual_physical_mapping *map ) {
00570         return hermon_cmd ( hermon,
00571                             HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM_AUX,
00572                                                 1, sizeof ( *map ) ),
00573                             0, map, 1, NULL );
00574 }
00575 
00576 static inline int
00577 hermon_cmd_set_icm_size ( struct hermon *hermon,
00578                           const struct hermonprm_scalar_parameter *icm_size,
00579                           struct hermonprm_scalar_parameter *icm_aux_size ) {
00580         return hermon_cmd ( hermon,
00581                             HERMON_HCR_INOUT_CMD ( HERMON_HCR_SET_ICM_SIZE,
00582                                                    0, sizeof ( *icm_size ),
00583                                                    0, sizeof (*icm_aux_size) ),
00584                             0, icm_size, 0, icm_aux_size );
00585 }
00586 
00587 static inline int
00588 hermon_cmd_unmap_fa ( struct hermon *hermon ) {
00589         return hermon_cmd ( hermon,
00590                             HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_FA ),
00591                             0, NULL, 0, NULL );
00592 }
00593 
00594 static inline int
00595 hermon_cmd_map_fa ( struct hermon *hermon,
00596                     const struct hermonprm_virtual_physical_mapping *map ) {
00597         return hermon_cmd ( hermon,
00598                             HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_FA,
00599                                                 1, sizeof ( *map ) ),
00600                             0, map, 1, NULL );
00601 }
00602 
00603 /***************************************************************************
00604  *
00605  * Memory translation table operations
00606  *
00607  ***************************************************************************
00608  */
00609 
00610 /**
00611  * Allocate MTT entries
00612  *
00613  * @v hermon            Hermon device
00614  * @v memory            Memory to map into MTT
00615  * @v len               Length of memory to map
00616  * @v mtt               MTT descriptor to fill in
00617  * @ret rc              Return status code
00618  */
00619 static int hermon_alloc_mtt ( struct hermon *hermon,
00620                               const void *memory, size_t len,
00621                               struct hermon_mtt *mtt ) {
00622         struct hermonprm_write_mtt write_mtt;
00623         physaddr_t start;
00624         physaddr_t addr;
00625         unsigned int page_offset;
00626         unsigned int num_pages;
00627         int mtt_offset;
00628         unsigned int mtt_base_addr;
00629         unsigned int i;
00630         int rc;
00631 
00632         /* Find available MTT entries */
00633         start = virt_to_phys ( memory );
00634         page_offset = ( start & ( HERMON_PAGE_SIZE - 1 ) );
00635         start -= page_offset;
00636         len += page_offset;
00637         num_pages = ( ( len + HERMON_PAGE_SIZE - 1 ) / HERMON_PAGE_SIZE );
00638         mtt_offset = hermon_bitmask_alloc ( hermon->mtt_inuse, HERMON_MAX_MTTS,
00639                                             num_pages );
00640         if ( mtt_offset < 0 ) {
00641                 DBGC ( hermon, "Hermon %p could not allocate %d MTT entries\n",
00642                        hermon, num_pages );
00643                 rc = mtt_offset;
00644                 goto err_mtt_offset;
00645         }
00646         mtt_base_addr = ( ( hermon->cap.reserved_mtts + mtt_offset ) *
00647                           hermon->cap.mtt_entry_size );
00648         addr = start;
00649 
00650         /* Fill in MTT structure */
00651         mtt->mtt_offset = mtt_offset;
00652         mtt->num_pages = num_pages;
00653         mtt->mtt_base_addr = mtt_base_addr;
00654         mtt->page_offset = page_offset;
00655 
00656         /* Construct and issue WRITE_MTT commands */
00657         for ( i = 0 ; i < num_pages ; i++ ) {
00658                 memset ( &write_mtt, 0, sizeof ( write_mtt ) );
00659                 MLX_FILL_1 ( &write_mtt.mtt_base_addr, 1,
00660                              value, mtt_base_addr );
00661                 MLX_FILL_H ( &write_mtt.mtt, 0, ptag_h, addr );
00662                 MLX_FILL_2 ( &write_mtt.mtt, 1,
00663                              p, 1,
00664                              ptag_l, ( addr >> 3 ) );
00665                 if ( ( rc = hermon_cmd_write_mtt ( hermon,
00666                                                    &write_mtt ) ) != 0 ) {
00667                         DBGC ( hermon, "Hermon %p could not write MTT at %x\n",
00668                                hermon, mtt_base_addr );
00669                         goto err_write_mtt;
00670                 }
00671                 addr += HERMON_PAGE_SIZE;
00672                 mtt_base_addr += hermon->cap.mtt_entry_size;
00673         }
00674 
00675         DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] for "
00676                "[%08lx,%08lx,%08lx,%08lx)\n", hermon, mtt->mtt_offset,
00677                ( mtt->mtt_offset + mtt->num_pages - 1 ), start,
00678                ( start + page_offset ), ( start + len ), addr );
00679 
00680         return 0;
00681 
00682  err_write_mtt:
00683         hermon_bitmask_free ( hermon->mtt_inuse, mtt_offset, num_pages );
00684  err_mtt_offset:
00685         return rc;
00686 }
00687 
00688 /**
00689  * Free MTT entries
00690  *
00691  * @v hermon            Hermon device
00692  * @v mtt               MTT descriptor
00693  */
00694 static void hermon_free_mtt ( struct hermon *hermon,
00695                               struct hermon_mtt *mtt ) {
00696 
00697         DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] freed\n",
00698                hermon, mtt->mtt_offset,
00699                ( mtt->mtt_offset + mtt->num_pages - 1 ) );
00700         hermon_bitmask_free ( hermon->mtt_inuse, mtt->mtt_offset,
00701                               mtt->num_pages );
00702 }
00703 
00704 /***************************************************************************
00705  *
00706  * Static configuration operations
00707  *
00708  ***************************************************************************
00709  */
00710 
00711 /**
00712  * Calculate offset within static configuration
00713  *
00714  * @v field             Field
00715  * @ret offset          Offset
00716  */
00717 #define HERMON_MOD_STAT_CFG_OFFSET( field )                                  \
00718         ( ( MLX_BIT_OFFSET ( struct hermonprm_mod_stat_cfg_st, field ) / 8 ) \
00719           & ~( sizeof ( struct hermonprm_scalar_parameter ) - 1 ) )
00720 
00721 /**
00722  * Query or modify static configuration
00723  *
00724  * @v hermon            Hermon device
00725  * @v port              Port
00726  * @v mode              Command mode
00727  * @v offset            Offset within static configuration
00728  * @v stat_cfg          Static configuration
00729  * @ret rc              Return status code
00730  */
00731 static int hermon_mod_stat_cfg ( struct hermon *hermon, unsigned int port,
00732                                  unsigned int mode, unsigned int offset,
00733                                  struct hermonprm_mod_stat_cfg *stat_cfg ) {
00734         struct hermonprm_scalar_parameter *portion =
00735                 ( ( void * ) &stat_cfg->u.bytes[offset] );
00736         struct hermonprm_mod_stat_cfg_input_mod mod;
00737         int rc;
00738 
00739         /* Sanity check */
00740         assert ( ( offset % sizeof ( *portion ) ) == 0 );
00741 
00742         /* Construct input modifier */
00743         memset ( &mod, 0, sizeof ( mod ) );
00744         MLX_FILL_2 ( &mod, 0,
00745                      portnum, port,
00746                      offset, offset );
00747 
00748         /* Issue command */
00749         if ( ( rc = hermon_cmd_mod_stat_cfg ( hermon, mode,
00750                                               be32_to_cpu ( mod.u.dwords[0] ),
00751                                               portion ) ) != 0 )
00752                 return rc;
00753 
00754         return 0;
00755 }
00756 
00757 /***************************************************************************
00758  *
00759  * MAD operations
00760  *
00761  ***************************************************************************
00762  */
00763 
00764 /**
00765  * Issue management datagram
00766  *
00767  * @v ibdev             Infiniband device
00768  * @v mad               Management datagram
00769  * @ret rc              Return status code
00770  */
00771 static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
00772         struct hermon *hermon = ib_get_drvdata ( ibdev );
00773         union hermonprm_mad mad_ifc;
00774         int rc;
00775 
00776         linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
00777                         mad_size_mismatch );
00778 
00779         /* Copy in request packet */
00780         memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
00781 
00782         /* Issue MAD */
00783         if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
00784                                          &mad_ifc ) ) != 0 ) {
00785                 DBGC ( hermon, "Hermon %p port %d could not issue MAD IFC: "
00786                        "%s\n", hermon, ibdev->port, strerror ( rc ) );
00787                 return rc;
00788         }
00789 
00790         /* Copy out reply packet */
00791         memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
00792 
00793         if ( mad->hdr.status != 0 ) {
00794                 DBGC ( hermon, "Hermon %p port %d MAD IFC status %04x\n",
00795                        hermon, ibdev->port, ntohs ( mad->hdr.status ) );
00796                 return -EIO;
00797         }
00798         return 0;
00799 }
00800 
00801 /***************************************************************************
00802  *
00803  * Completion queue operations
00804  *
00805  ***************************************************************************
00806  */
00807 
00808 /**
00809  * Dump completion queue context (for debugging only)
00810  *
00811  * @v hermon            Hermon device
00812  * @v cq                Completion queue
00813  * @ret rc              Return status code
00814  */
00815 static __attribute__ (( unused )) int
00816 hermon_dump_cqctx ( struct hermon *hermon, struct ib_completion_queue *cq ) {
00817         struct hermonprm_completion_queue_context cqctx;
00818         int rc;
00819 
00820         memset ( &cqctx, 0, sizeof ( cqctx ) );
00821         if ( ( rc = hermon_cmd_query_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
00822                 DBGC ( hermon, "Hermon %p CQN %#lx QUERY_CQ failed: %s\n",
00823                        hermon, cq->cqn, strerror ( rc ) );
00824                 return rc;
00825         }
00826         DBGC ( hermon, "Hermon %p CQN %#lx context:\n", hermon, cq->cqn );
00827         DBGC_HDA ( hermon, 0, &cqctx, sizeof ( cqctx ) );
00828 
00829         return 0;
00830 }
00831 
00832 /**
00833  * Create completion queue
00834  *
00835  * @v ibdev             Infiniband device
00836  * @v cq                Completion queue
00837  * @ret rc              Return status code
00838  */
00839 static int hermon_create_cq ( struct ib_device *ibdev,
00840                               struct ib_completion_queue *cq ) {
00841         struct hermon *hermon = ib_get_drvdata ( ibdev );
00842         struct hermon_completion_queue *hermon_cq;
00843         struct hermonprm_completion_queue_context cqctx;
00844         int cqn_offset;
00845         unsigned int i;
00846         int rc;
00847 
00848         /* Find a free completion queue number */
00849         cqn_offset = hermon_bitmask_alloc ( hermon->cq_inuse,
00850                                             HERMON_MAX_CQS, 1 );
00851         if ( cqn_offset < 0 ) {
00852                 DBGC ( hermon, "Hermon %p out of completion queues\n",
00853                        hermon );
00854                 rc = cqn_offset;
00855                 goto err_cqn_offset;
00856         }
00857         cq->cqn = ( hermon->cap.reserved_cqs + cqn_offset );
00858 
00859         /* Allocate control structures */
00860         hermon_cq = zalloc ( sizeof ( *hermon_cq ) );
00861         if ( ! hermon_cq ) {
00862                 rc = -ENOMEM;
00863                 goto err_hermon_cq;
00864         }
00865 
00866         /* Allocate doorbell */
00867         hermon_cq->doorbell = malloc_dma ( sizeof ( hermon_cq->doorbell[0] ),
00868                                            sizeof ( hermon_cq->doorbell[0] ) );
00869         if ( ! hermon_cq->doorbell ) {
00870                 rc = -ENOMEM;
00871                 goto err_doorbell;
00872         }
00873         memset ( hermon_cq->doorbell, 0, sizeof ( hermon_cq->doorbell[0] ) );
00874 
00875         /* Allocate completion queue itself */
00876         hermon_cq->cqe_size = ( cq->num_cqes * sizeof ( hermon_cq->cqe[0] ) );
00877         hermon_cq->cqe = malloc_dma ( hermon_cq->cqe_size,
00878                                       sizeof ( hermon_cq->cqe[0] ) );
00879         if ( ! hermon_cq->cqe ) {
00880                 rc = -ENOMEM;
00881                 goto err_cqe;
00882         }
00883         memset ( hermon_cq->cqe, 0, hermon_cq->cqe_size );
00884         for ( i = 0 ; i < cq->num_cqes ; i++ ) {
00885                 MLX_FILL_1 ( &hermon_cq->cqe[i].normal, 7, owner, 1 );
00886         }
00887         barrier();
00888 
00889         /* Allocate MTT entries */
00890         if ( ( rc = hermon_alloc_mtt ( hermon, hermon_cq->cqe,
00891                                        hermon_cq->cqe_size,
00892                                        &hermon_cq->mtt ) ) != 0 )
00893                 goto err_alloc_mtt;
00894 
00895         /* Hand queue over to hardware */
00896         memset ( &cqctx, 0, sizeof ( cqctx ) );
00897         MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
00898         MLX_FILL_1 ( &cqctx, 2,
00899                      page_offset, ( hermon_cq->mtt.page_offset >> 5 ) );
00900         MLX_FILL_2 ( &cqctx, 3,
00901                      usr_page, HERMON_UAR_NON_EQ_PAGE,
00902                      log_cq_size, fls ( cq->num_cqes - 1 ) );
00903         MLX_FILL_1 ( &cqctx, 5, c_eqn, hermon->eq.eqn );
00904         MLX_FILL_H ( &cqctx, 6, mtt_base_addr_h,
00905                      hermon_cq->mtt.mtt_base_addr );
00906         MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l,
00907                      ( hermon_cq->mtt.mtt_base_addr >> 3 ) );
00908         MLX_FILL_H ( &cqctx, 14, db_record_addr_h,
00909                      virt_to_phys ( hermon_cq->doorbell ) );
00910         MLX_FILL_1 ( &cqctx, 15, db_record_addr_l,
00911                      ( virt_to_phys ( hermon_cq->doorbell ) >> 3 ) );
00912         if ( ( rc = hermon_cmd_sw2hw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
00913                 DBGC ( hermon, "Hermon %p CQN %#lx SW2HW_CQ failed: %s\n",
00914                        hermon, cq->cqn, strerror ( rc ) );
00915                 goto err_sw2hw_cq;
00916         }
00917 
00918         DBGC ( hermon, "Hermon %p CQN %#lx ring [%08lx,%08lx), doorbell "
00919                "%08lx\n", hermon, cq->cqn, virt_to_phys ( hermon_cq->cqe ),
00920                ( virt_to_phys ( hermon_cq->cqe ) + hermon_cq->cqe_size ),
00921                virt_to_phys ( hermon_cq->doorbell ) );
00922         ib_cq_set_drvdata ( cq, hermon_cq );
00923         return 0;
00924 
00925  err_sw2hw_cq:
00926         hermon_free_mtt ( hermon, &hermon_cq->mtt );
00927  err_alloc_mtt:
00928         free_dma ( hermon_cq->cqe, hermon_cq->cqe_size );
00929  err_cqe:
00930         free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
00931  err_doorbell:
00932         free ( hermon_cq );
00933  err_hermon_cq:
00934         hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
00935  err_cqn_offset:
00936         return rc;
00937 }
00938 
00939 /**
00940  * Destroy completion queue
00941  *
00942  * @v ibdev             Infiniband device
00943  * @v cq                Completion queue
00944  */
00945 static void hermon_destroy_cq ( struct ib_device *ibdev,
00946                                 struct ib_completion_queue *cq ) {
00947         struct hermon *hermon = ib_get_drvdata ( ibdev );
00948         struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
00949         struct hermonprm_completion_queue_context cqctx;
00950         int cqn_offset;
00951         int rc;
00952 
00953         /* Take ownership back from hardware */
00954         if ( ( rc = hermon_cmd_hw2sw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
00955                 DBGC ( hermon, "Hermon %p CQN %#lx FATAL HW2SW_CQ failed: "
00956                        "%s\n", hermon, cq->cqn, strerror ( rc ) );
00957                 /* Leak memory and return; at least we avoid corruption */
00958                 return;
00959         }
00960 
00961         /* Free MTT entries */
00962         hermon_free_mtt ( hermon, &hermon_cq->mtt );
00963 
00964         /* Free memory */
00965         free_dma ( hermon_cq->cqe, hermon_cq->cqe_size );
00966         free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
00967         free ( hermon_cq );
00968 
00969         /* Mark queue number as free */
00970         cqn_offset = ( cq->cqn - hermon->cap.reserved_cqs );
00971         hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
00972 
00973         ib_cq_set_drvdata ( cq, NULL );
00974 }
00975 
00976 /***************************************************************************
00977  *
00978  * Queue pair operations
00979  *
00980  ***************************************************************************
00981  */
00982 
00983 /**
00984  * Assign queue pair number
00985  *
00986  * @v ibdev             Infiniband device
00987  * @v qp                Queue pair
00988  * @ret rc              Return status code
00989  */
00990 static int hermon_alloc_qpn ( struct ib_device *ibdev,
00991                               struct ib_queue_pair *qp ) {
00992         struct hermon *hermon = ib_get_drvdata ( ibdev );
00993         unsigned int port_offset;
00994         int qpn_offset;
00995 
00996         /* Calculate queue pair number */
00997         port_offset = ( ibdev->port - HERMON_PORT_BASE );
00998 
00999         switch ( qp->type ) {
01000         case IB_QPT_SMI:
01001                 qp->qpn = ( hermon->special_qpn_base + port_offset );
01002                 return 0;
01003         case IB_QPT_GSI:
01004                 qp->qpn = ( hermon->special_qpn_base + 2 + port_offset );
01005                 return 0;
01006         case IB_QPT_UD:
01007         case IB_QPT_RC:
01008         case IB_QPT_ETH:
01009                 /* Find a free queue pair number */
01010                 qpn_offset = hermon_bitmask_alloc ( hermon->qp_inuse,
01011                                                     HERMON_MAX_QPS, 1 );
01012                 if ( qpn_offset < 0 ) {
01013                         DBGC ( hermon, "Hermon %p out of queue pairs\n",
01014                                hermon );
01015                         return qpn_offset;
01016                 }
01017                 qp->qpn = ( ( random() & HERMON_QPN_RANDOM_MASK ) |
01018                             ( hermon->qpn_base + qpn_offset ) );
01019                 return 0;
01020         default:
01021                 DBGC ( hermon, "Hermon %p unsupported QP type %d\n",
01022                        hermon, qp->type );
01023                 return -ENOTSUP;
01024         }
01025 }
01026 
01027 /**
01028  * Free queue pair number
01029  *
01030  * @v ibdev             Infiniband device
01031  * @v qp                Queue pair
01032  */
01033 static void hermon_free_qpn ( struct ib_device *ibdev,
01034                               struct ib_queue_pair *qp ) {
01035         struct hermon *hermon = ib_get_drvdata ( ibdev );
01036         int qpn_offset;
01037 
01038         qpn_offset = ( ( qp->qpn & ~HERMON_QPN_RANDOM_MASK )
01039                        - hermon->qpn_base );
01040         if ( qpn_offset >= 0 )
01041                 hermon_bitmask_free ( hermon->qp_inuse, qpn_offset, 1 );
01042 }
01043 
01044 /**
01045  * Calculate transmission rate
01046  *
01047  * @v av                Address vector
01048  * @ret hermon_rate     Hermon rate
01049  */
01050 static unsigned int hermon_rate ( struct ib_address_vector *av ) {
01051         return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) )
01052                  ? ( av->rate + 5 ) : 0 );
01053 }
01054 
01055 /**
01056  * Calculate schedule queue
01057  *
01058  * @v ibdev             Infiniband device
01059  * @v qp                Queue pair
01060  * @ret sched_queue     Schedule queue
01061  */
01062 static unsigned int hermon_sched_queue ( struct ib_device *ibdev,
01063                                          struct ib_queue_pair *qp ) {
01064         return ( ( ( qp->type == IB_QPT_SMI ) ?
01065                    HERMON_SCHED_QP0 : HERMON_SCHED_DEFAULT ) |
01066                  ( ( ibdev->port - 1 ) << 6 ) );
01067 }
01068 
01069 /** Queue pair transport service type map */
01070 static uint8_t hermon_qp_st[] = {
01071         [IB_QPT_SMI] = HERMON_ST_MLX,
01072         [IB_QPT_GSI] = HERMON_ST_MLX,
01073         [IB_QPT_UD] = HERMON_ST_UD,
01074         [IB_QPT_RC] = HERMON_ST_RC,
01075         [IB_QPT_ETH] = HERMON_ST_MLX,
01076 };
01077 
01078 /**
01079  * Dump queue pair context (for debugging only)
01080  *
01081  * @v hermon            Hermon device
01082  * @v qp                Queue pair
01083  * @ret rc              Return status code
01084  */
01085 static __attribute__ (( unused )) int
01086 hermon_dump_qpctx ( struct hermon *hermon, struct ib_queue_pair *qp ) {
01087         struct hermonprm_qp_ee_state_transitions qpctx;
01088         int rc;
01089 
01090         memset ( &qpctx, 0, sizeof ( qpctx ) );
01091         if ( ( rc = hermon_cmd_query_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ) {
01092                 DBGC ( hermon, "Hermon %p QPN %#lx QUERY_QP failed: %s\n",
01093                        hermon, qp->qpn, strerror ( rc ) );
01094                 return rc;
01095         }
01096         DBGC ( hermon, "Hermon %p QPN %#lx context:\n", hermon, qp->qpn );
01097         DBGC_HDA ( hermon, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) );
01098 
01099         return 0;
01100 }
01101 
01102 /**
01103  * Create queue pair
01104  *
01105  * @v ibdev             Infiniband device
01106  * @v qp                Queue pair
01107  * @ret rc              Return status code
01108  */
01109 static int hermon_create_qp ( struct ib_device *ibdev,
01110                               struct ib_queue_pair *qp ) {
01111         struct hermon *hermon = ib_get_drvdata ( ibdev );
01112         struct hermon_queue_pair *hermon_qp;
01113         struct hermonprm_qp_ee_state_transitions qpctx;
01114         struct hermonprm_wqe_segment_data_ptr *data;
01115         unsigned int i;
01116         int rc;
01117 
01118         /* Calculate queue pair number */
01119         if ( ( rc = hermon_alloc_qpn ( ibdev, qp ) ) != 0 )
01120                 goto err_alloc_qpn;
01121 
01122         /* Allocate control structures */
01123         hermon_qp = zalloc ( sizeof ( *hermon_qp ) );
01124         if ( ! hermon_qp ) {
01125                 rc = -ENOMEM;
01126                 goto err_hermon_qp;
01127         }
01128 
01129         /* Allocate doorbells */
01130         hermon_qp->recv.doorbell =
01131                 malloc_dma ( sizeof ( hermon_qp->recv.doorbell[0] ),
01132                              sizeof ( hermon_qp->recv.doorbell[0] ) );
01133         if ( ! hermon_qp->recv.doorbell ) {
01134                 rc = -ENOMEM;
01135                 goto err_recv_doorbell;
01136         }
01137         memset ( hermon_qp->recv.doorbell, 0,
01138                  sizeof ( hermon_qp->recv.doorbell[0] ) );
01139         hermon_qp->send.doorbell =
01140                 ( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE +
01141                   HERMON_DB_POST_SND_OFFSET );
01142 
01143         /* Allocate work queue buffer */
01144         hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 +
01145                                 ( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) );
01146         hermon_qp->send.num_wqes =
01147                 ( 1 << fls ( hermon_qp->send.num_wqes - 1 ) ); /* round up */
01148         hermon_qp->send.wqe_size = ( hermon_qp->send.num_wqes *
01149                                      sizeof ( hermon_qp->send.wqe[0] ) );
01150         hermon_qp->recv.wqe_size = ( qp->recv.num_wqes *
01151                                      sizeof ( hermon_qp->recv.wqe[0] ) );
01152         if ( ( qp->type == IB_QPT_SMI ) || ( qp->type == IB_QPT_GSI ) ||
01153              ( qp->type == IB_QPT_UD ) ) {
01154                 hermon_qp->recv.grh_size = ( qp->recv.num_wqes *
01155                                              sizeof ( hermon_qp->recv.grh[0] ));
01156         }
01157         hermon_qp->wqe_size = ( hermon_qp->send.wqe_size +
01158                                 hermon_qp->recv.wqe_size +
01159                                 hermon_qp->recv.grh_size );
01160         hermon_qp->wqe = malloc_dma ( hermon_qp->wqe_size,
01161                                       sizeof ( hermon_qp->send.wqe[0] ) );
01162         if ( ! hermon_qp->wqe ) {
01163                 rc = -ENOMEM;
01164                 goto err_alloc_wqe;
01165         }
01166         hermon_qp->send.wqe = hermon_qp->wqe;
01167         hermon_qp->recv.wqe = ( hermon_qp->wqe + hermon_qp->send.wqe_size );
01168         if ( hermon_qp->recv.grh_size ) {
01169                 hermon_qp->recv.grh = ( hermon_qp->wqe +
01170                                         hermon_qp->send.wqe_size +
01171                                         hermon_qp->recv.wqe_size );
01172         }
01173 
01174         /* Initialise work queue entries */
01175         memset ( hermon_qp->send.wqe, 0xff, hermon_qp->send.wqe_size );
01176         memset ( hermon_qp->recv.wqe, 0, hermon_qp->recv.wqe_size );
01177         data = &hermon_qp->recv.wqe[0].recv.data[0];
01178         for ( i = 0 ; i < ( hermon_qp->recv.wqe_size / sizeof ( *data ) ); i++){
01179                 MLX_FILL_1 ( data, 1, l_key, HERMON_INVALID_LKEY );
01180                 data++;
01181         }
01182 
01183         /* Allocate MTT entries */
01184         if ( ( rc = hermon_alloc_mtt ( hermon, hermon_qp->wqe,
01185                                        hermon_qp->wqe_size,
01186                                        &hermon_qp->mtt ) ) != 0 ) {
01187                 goto err_alloc_mtt;
01188         }
01189 
01190         /* Transition queue to INIT state */
01191         memset ( &qpctx, 0, sizeof ( qpctx ) );
01192         MLX_FILL_2 ( &qpctx, 2,
01193                      qpc_eec_data.pm_state, HERMON_PM_STATE_MIGRATED,
01194                      qpc_eec_data.st, hermon_qp_st[qp->type] );
01195         MLX_FILL_1 ( &qpctx, 3, qpc_eec_data.pd, HERMON_GLOBAL_PD );
01196         MLX_FILL_4 ( &qpctx, 4,
01197                      qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
01198                      qpc_eec_data.log_rq_stride,
01199                      ( fls ( sizeof ( hermon_qp->recv.wqe[0] ) - 1 ) - 4 ),
01200                      qpc_eec_data.log_sq_size,
01201                      fls ( hermon_qp->send.num_wqes - 1 ),
01202                      qpc_eec_data.log_sq_stride,
01203                      ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) );
01204         MLX_FILL_1 ( &qpctx, 5,
01205                      qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE );
01206         MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
01207         MLX_FILL_4 ( &qpctx, 38,
01208                      qpc_eec_data.rre, 1,
01209                      qpc_eec_data.rwe, 1,
01210                      qpc_eec_data.rae, 1,
01211                      qpc_eec_data.page_offset,
01212                      ( hermon_qp->mtt.page_offset >> 6 ) );
01213         MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
01214         MLX_FILL_H ( &qpctx, 42, qpc_eec_data.db_record_addr_h,
01215                      virt_to_phys ( hermon_qp->recv.doorbell ) );
01216         MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l,
01217                      ( virt_to_phys ( hermon_qp->recv.doorbell ) >> 2 ) );
01218         MLX_FILL_H ( &qpctx, 52, qpc_eec_data.mtt_base_addr_h,
01219                      hermon_qp->mtt.mtt_base_addr );
01220         MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l,
01221                      ( hermon_qp->mtt.mtt_base_addr >> 3 ) );
01222         if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn,
01223                                              &qpctx ) ) != 0 ) {
01224                 DBGC ( hermon, "Hermon %p QPN %#lx RST2INIT_QP failed: %s\n",
01225                        hermon, qp->qpn, strerror ( rc ) );
01226                 goto err_rst2init_qp;
01227         }
01228         hermon_qp->state = HERMON_QP_ST_INIT;
01229 
01230         DBGC ( hermon, "Hermon %p QPN %#lx send ring [%08lx,%08lx), doorbell "
01231                "%08lx\n", hermon, qp->qpn,
01232                virt_to_phys ( hermon_qp->send.wqe ),
01233                ( virt_to_phys ( hermon_qp->send.wqe ) +
01234                  hermon_qp->send.wqe_size ),
01235                virt_to_phys ( hermon_qp->send.doorbell ) );
01236         DBGC ( hermon, "Hermon %p QPN %#lx receive ring [%08lx,%08lx), "
01237                "doorbell %08lx\n", hermon, qp->qpn,
01238                virt_to_phys ( hermon_qp->recv.wqe ),
01239                ( virt_to_phys ( hermon_qp->recv.wqe ) +
01240                  hermon_qp->recv.wqe_size ),
01241                virt_to_phys ( hermon_qp->recv.doorbell ) );
01242         DBGC ( hermon, "Hermon %p QPN %#lx send CQN %#lx receive CQN %#lx\n",
01243                hermon, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn );
01244         ib_qp_set_drvdata ( qp, hermon_qp );
01245         return 0;
01246 
01247         hermon_cmd_2rst_qp ( hermon, qp->qpn );
01248  err_rst2init_qp:
01249         hermon_free_mtt ( hermon, &hermon_qp->mtt );
01250  err_alloc_mtt:
01251         free_dma ( hermon_qp->wqe, hermon_qp->wqe_size );
01252  err_alloc_wqe:
01253         free_dma ( hermon_qp->recv.doorbell,
01254                    sizeof ( hermon_qp->recv.doorbell[0] ) );
01255  err_recv_doorbell:
01256         free ( hermon_qp );
01257  err_hermon_qp:
01258         hermon_free_qpn ( ibdev, qp );
01259  err_alloc_qpn:
01260         return rc;
01261 }
01262 
01263 /**
01264  * Modify queue pair
01265  *
01266  * @v ibdev             Infiniband device
01267  * @v qp                Queue pair
01268  * @ret rc              Return status code
01269  */
01270 static int hermon_modify_qp ( struct ib_device *ibdev,
01271                               struct ib_queue_pair *qp ) {
01272         struct hermon *hermon = ib_get_drvdata ( ibdev );
01273         struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
01274         struct hermonprm_qp_ee_state_transitions qpctx;
01275         int rc;
01276 
01277         /* Transition queue to RTR state, if applicable */
01278         if ( hermon_qp->state < HERMON_QP_ST_RTR ) {
01279                 memset ( &qpctx, 0, sizeof ( qpctx ) );
01280                 MLX_FILL_2 ( &qpctx, 4,
01281                              qpc_eec_data.mtu,
01282                              ( ( qp->type == IB_QPT_ETH ) ?
01283                                HERMON_MTU_ETH : HERMON_MTU_2048 ),
01284                              qpc_eec_data.msg_max, 31 );
01285                 MLX_FILL_1 ( &qpctx, 7,
01286                              qpc_eec_data.remote_qpn_een, qp->av.qpn );
01287                 MLX_FILL_1 ( &qpctx, 9,
01288                              qpc_eec_data.primary_address_path.rlid,
01289                              qp->av.lid );
01290                 MLX_FILL_1 ( &qpctx, 10,
01291                              qpc_eec_data.primary_address_path.max_stat_rate,
01292                              hermon_rate ( &qp->av ) );
01293                 memcpy ( &qpctx.u.dwords[12], &qp->av.gid,
01294                          sizeof ( qp->av.gid ) );
01295                 MLX_FILL_1 ( &qpctx, 16,
01296                              qpc_eec_data.primary_address_path.sched_queue,
01297                              hermon_sched_queue ( ibdev, qp ) );
01298                 MLX_FILL_1 ( &qpctx, 39,
01299                              qpc_eec_data.next_rcv_psn, qp->recv.psn );
01300                 if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn,
01301                                                      &qpctx ) ) != 0 ) {
01302                         DBGC ( hermon, "Hermon %p QPN %#lx INIT2RTR_QP failed:"
01303                                " %s\n", hermon, qp->qpn, strerror ( rc ) );
01304                         return rc;
01305                 }
01306                 hermon_qp->state = HERMON_QP_ST_RTR;
01307         }
01308 
01309         /* Transition queue to RTS state */
01310         if ( hermon_qp->state < HERMON_QP_ST_RTS ) {
01311                 memset ( &qpctx, 0, sizeof ( qpctx ) );
01312                 MLX_FILL_1 ( &qpctx, 10,
01313                              qpc_eec_data.primary_address_path.ack_timeout,
01314                              14 /* 4.096us * 2^(14) = 67ms */ );
01315                 MLX_FILL_2 ( &qpctx, 30,
01316                              qpc_eec_data.retry_count, HERMON_RETRY_MAX,
01317                              qpc_eec_data.rnr_retry, HERMON_RETRY_MAX );
01318                 MLX_FILL_1 ( &qpctx, 32,
01319                              qpc_eec_data.next_send_psn, qp->send.psn );
01320                 if ( ( rc = hermon_cmd_rtr2rts_qp ( hermon, qp->qpn,
01321                                                     &qpctx ) ) != 0 ) {
01322                         DBGC ( hermon, "Hermon %p QPN %#lx RTR2RTS_QP failed: "
01323                                "%s\n", hermon, qp->qpn, strerror ( rc ) );
01324                         return rc;
01325                 }
01326                 hermon_qp->state = HERMON_QP_ST_RTS;
01327         }
01328 
01329         /* Update parameters in RTS state */
01330         memset ( &qpctx, 0, sizeof ( qpctx ) );
01331         MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY );
01332         MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
01333         if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
01334                 DBGC ( hermon, "Hermon %p QPN %#lx RTS2RTS_QP failed: %s\n",
01335                        hermon, qp->qpn, strerror ( rc ) );
01336                 return rc;
01337         }
01338 
01339         return 0;
01340 }
01341 
01342 /**
01343  * Destroy queue pair
01344  *
01345  * @v ibdev             Infiniband device
01346  * @v qp                Queue pair
01347  */
01348 static void hermon_destroy_qp ( struct ib_device *ibdev,
01349                                 struct ib_queue_pair *qp ) {
01350         struct hermon *hermon = ib_get_drvdata ( ibdev );
01351         struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
01352         int rc;
01353 
01354         /* Take ownership back from hardware */
01355         if ( ( rc = hermon_cmd_2rst_qp ( hermon, qp->qpn ) ) != 0 ) {
01356                 DBGC ( hermon, "Hermon %p QPN %#lx FATAL 2RST_QP failed: %s\n",
01357                        hermon, qp->qpn, strerror ( rc ) );
01358                 /* Leak memory and return; at least we avoid corruption */
01359                 return;
01360         }
01361 
01362         /* Free MTT entries */
01363         hermon_free_mtt ( hermon, &hermon_qp->mtt );
01364 
01365         /* Free memory */
01366         free_dma ( hermon_qp->wqe, hermon_qp->wqe_size );
01367         free_dma ( hermon_qp->recv.doorbell,
01368                    sizeof ( hermon_qp->recv.doorbell[0] ) );
01369         free ( hermon_qp );
01370 
01371         /* Mark queue number as free */
01372         hermon_free_qpn ( ibdev, qp );
01373 
01374         ib_qp_set_drvdata ( qp, NULL );
01375 }
01376 
01377 /***************************************************************************
01378  *
01379  * Work request operations
01380  *
01381  ***************************************************************************
01382  */
01383 
01384 /**
01385  * Construct UD send work queue entry
01386  *
01387  * @v ibdev             Infiniband device
01388  * @v qp                Queue pair
01389  * @v dest              Destination address vector
01390  * @v iobuf             I/O buffer
01391  * @v wqe               Send work queue entry
01392  * @ret opcode          Control opcode
01393  */
01394 static __attribute__ (( unused )) unsigned int
01395 hermon_fill_nop_send_wqe ( struct ib_device *ibdev __unused,
01396                            struct ib_queue_pair *qp __unused,
01397                            struct ib_address_vector *dest __unused,
01398                            struct io_buffer *iobuf __unused,
01399                            union hermon_send_wqe *wqe ) {
01400 
01401         MLX_FILL_1 ( &wqe->ctrl, 1, ds, ( sizeof ( wqe->ctrl ) / 16 ) );
01402         MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ );
01403         return HERMON_OPCODE_NOP;
01404 }
01405 
01406 /**
01407  * Construct UD send work queue entry
01408  *
01409  * @v ibdev             Infiniband device
01410  * @v qp                Queue pair
01411  * @v dest              Destination address vector
01412  * @v iobuf             I/O buffer
01413  * @v wqe               Send work queue entry
01414  * @ret opcode          Control opcode
01415  */
01416 static unsigned int
01417 hermon_fill_ud_send_wqe ( struct ib_device *ibdev,
01418                           struct ib_queue_pair *qp __unused,
01419                           struct ib_address_vector *dest,
01420                           struct io_buffer *iobuf,
01421                           union hermon_send_wqe *wqe ) {
01422         struct hermon *hermon = ib_get_drvdata ( ibdev );
01423 
01424         MLX_FILL_1 ( &wqe->ud.ctrl, 1, ds,
01425                      ( ( offsetof ( typeof ( wqe->ud ), data[1] ) / 16 ) ) );
01426         MLX_FILL_1 ( &wqe->ud.ctrl, 2, c, 0x03 /* generate completion */ );
01427         MLX_FILL_2 ( &wqe->ud.ud, 0,
01428                      ud_address_vector.pd, HERMON_GLOBAL_PD,
01429                      ud_address_vector.port_number, ibdev->port );
01430         MLX_FILL_2 ( &wqe->ud.ud, 1,
01431                      ud_address_vector.rlid, dest->lid,
01432                      ud_address_vector.g, dest->gid_present );
01433         MLX_FILL_1 ( &wqe->ud.ud, 2,
01434                      ud_address_vector.max_stat_rate, hermon_rate ( dest ) );
01435         MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, dest->sl );
01436         memcpy ( &wqe->ud.ud.u.dwords[4], &dest->gid, sizeof ( dest->gid ) );
01437         MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, dest->qpn );
01438         MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, dest->qkey );
01439         MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) );
01440         MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, hermon->lkey );
01441         MLX_FILL_H ( &wqe->ud.data[0], 2,
01442                      local_address_h, virt_to_bus ( iobuf->data ) );
01443         MLX_FILL_1 ( &wqe->ud.data[0], 3,
01444                      local_address_l, virt_to_bus ( iobuf->data ) );
01445         return HERMON_OPCODE_SEND;
01446 }
01447 
01448 /**
01449  * Construct MLX send work queue entry
01450  *
01451  * @v ibdev             Infiniband device
01452  * @v qp                Queue pair
01453  * @v dest              Destination address vector
01454  * @v iobuf             I/O buffer
01455  * @v wqe               Send work queue entry
01456  * @ret opcode          Control opcode
01457  */
01458 static unsigned int
01459 hermon_fill_mlx_send_wqe ( struct ib_device *ibdev,
01460                            struct ib_queue_pair *qp,
01461                            struct ib_address_vector *dest,
01462                            struct io_buffer *iobuf,
01463                            union hermon_send_wqe *wqe ) {
01464         struct hermon *hermon = ib_get_drvdata ( ibdev );
01465         struct io_buffer headers;
01466 
01467         /* Construct IB headers */
01468         iob_populate ( &headers, &wqe->mlx.headers, 0,
01469                        sizeof ( wqe->mlx.headers ) );
01470         iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) );
01471         ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
01472 
01473         /* Fill work queue entry */
01474         MLX_FILL_1 ( &wqe->mlx.ctrl, 1, ds,
01475                      ( ( offsetof ( typeof ( wqe->mlx ), data[2] ) / 16 ) ) );
01476         MLX_FILL_5 ( &wqe->mlx.ctrl, 2,
01477                      c, 0x03 /* generate completion */,
01478                      icrc, 0 /* generate ICRC */,
01479                      max_statrate, hermon_rate ( dest ),
01480                      slr, 0,
01481                      v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) );
01482         MLX_FILL_1 ( &wqe->mlx.ctrl, 3, rlid, dest->lid );
01483         MLX_FILL_1 ( &wqe->mlx.data[0], 0,
01484                      byte_count, iob_len ( &headers ) );
01485         MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, hermon->lkey );
01486         MLX_FILL_H ( &wqe->mlx.data[0], 2,
01487                      local_address_h, virt_to_bus ( headers.data ) );
01488         MLX_FILL_1 ( &wqe->mlx.data[0], 3,
01489                      local_address_l, virt_to_bus ( headers.data ) );
01490         MLX_FILL_1 ( &wqe->mlx.data[1], 0,
01491                      byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) );
01492         MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, hermon->lkey );
01493         MLX_FILL_H ( &wqe->mlx.data[1], 2,
01494                      local_address_h, virt_to_bus ( iobuf->data ) );
01495         MLX_FILL_1 ( &wqe->mlx.data[1], 3,
01496                      local_address_l, virt_to_bus ( iobuf->data ) );
01497         return HERMON_OPCODE_SEND;
01498 }
01499 
01500 /**
01501  * Construct RC send work queue entry
01502  *
01503  * @v ibdev             Infiniband device
01504  * @v qp                Queue pair
01505  * @v dest              Destination address vector
01506  * @v iobuf             I/O buffer
01507  * @v wqe               Send work queue entry
01508  * @ret opcode          Control opcode
01509  */
01510 static unsigned int
01511 hermon_fill_rc_send_wqe ( struct ib_device *ibdev,
01512                           struct ib_queue_pair *qp __unused,
01513                           struct ib_address_vector *dest __unused,
01514                           struct io_buffer *iobuf,
01515                           union hermon_send_wqe *wqe ) {
01516         struct hermon *hermon = ib_get_drvdata ( ibdev );
01517 
01518         MLX_FILL_1 ( &wqe->rc.ctrl, 1, ds,
01519                      ( ( offsetof ( typeof ( wqe->rc ), data[1] ) / 16 ) ) );
01520         MLX_FILL_1 ( &wqe->rc.ctrl, 2, c, 0x03 /* generate completion */ );
01521         MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) );
01522         MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, hermon->lkey );
01523         MLX_FILL_H ( &wqe->rc.data[0], 2,
01524                      local_address_h, virt_to_bus ( iobuf->data ) );
01525         MLX_FILL_1 ( &wqe->rc.data[0], 3,
01526                      local_address_l, virt_to_bus ( iobuf->data ) );
01527         return HERMON_OPCODE_SEND;
01528 }
01529 
01530 /**
01531  * Construct Ethernet send work queue entry
01532  *
01533  * @v ibdev             Infiniband device
01534  * @v qp                Queue pair
01535  * @v dest              Destination address vector
01536  * @v iobuf             I/O buffer
01537  * @v wqe               Send work queue entry
01538  * @ret opcode          Control opcode
01539  */
01540 static unsigned int
01541 hermon_fill_eth_send_wqe ( struct ib_device *ibdev,
01542                            struct ib_queue_pair *qp __unused,
01543                            struct ib_address_vector *dest __unused,
01544                            struct io_buffer *iobuf,
01545                            union hermon_send_wqe *wqe ) {
01546         struct hermon *hermon = ib_get_drvdata ( ibdev );
01547 
01548         /* Fill work queue entry */
01549         MLX_FILL_1 ( &wqe->eth.ctrl, 1, ds,
01550                      ( ( offsetof ( typeof ( wqe->mlx ), data[1] ) / 16 ) ) );
01551         MLX_FILL_2 ( &wqe->eth.ctrl, 2,
01552                      c, 0x03 /* generate completion */,
01553                      s, 1 /* inhibit ICRC */ );
01554         MLX_FILL_1 ( &wqe->eth.data[0], 0,
01555                      byte_count, iob_len ( iobuf ) );
01556         MLX_FILL_1 ( &wqe->eth.data[0], 1, l_key, hermon->lkey );
01557         MLX_FILL_H ( &wqe->eth.data[0], 2,
01558                      local_address_h, virt_to_bus ( iobuf->data ) );
01559         MLX_FILL_1 ( &wqe->eth.data[0], 3,
01560                      local_address_l, virt_to_bus ( iobuf->data ) );
01561         return HERMON_OPCODE_SEND;
01562 }
01563 
01564 /** Work queue entry constructors */
01565 static unsigned int
01566 ( * hermon_fill_send_wqe[] ) ( struct ib_device *ibdev,
01567                                struct ib_queue_pair *qp,
01568                                struct ib_address_vector *dest,
01569                                struct io_buffer *iobuf,
01570                                union hermon_send_wqe *wqe ) = {
01571         [IB_QPT_SMI] = hermon_fill_mlx_send_wqe,
01572         [IB_QPT_GSI] = hermon_fill_mlx_send_wqe,
01573         [IB_QPT_UD] = hermon_fill_ud_send_wqe,
01574         [IB_QPT_RC] = hermon_fill_rc_send_wqe,
01575         [IB_QPT_ETH] = hermon_fill_eth_send_wqe,
01576 };
01577 
01578 /**
01579  * Post send work queue entry
01580  *
01581  * @v ibdev             Infiniband device
01582  * @v qp                Queue pair
01583  * @v dest              Destination address vector
01584  * @v iobuf             I/O buffer
01585  * @ret rc              Return status code
01586  */
01587 static int hermon_post_send ( struct ib_device *ibdev,
01588                               struct ib_queue_pair *qp,
01589                               struct ib_address_vector *dest,
01590                               struct io_buffer *iobuf ) {
01591         struct hermon *hermon = ib_get_drvdata ( ibdev );
01592         struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
01593         struct ib_work_queue *wq = &qp->send;
01594         struct hermon_send_work_queue *hermon_send_wq = &hermon_qp->send;
01595         union hermon_send_wqe *wqe;
01596         union hermonprm_doorbell_register db_reg;
01597         unsigned long wqe_idx_mask;
01598         unsigned long wqe_idx;
01599         unsigned int owner;
01600         unsigned int opcode;
01601 
01602         /* Allocate work queue entry */
01603         wqe_idx = ( wq->next_idx & ( hermon_send_wq->num_wqes - 1 ) );
01604         owner = ( ( wq->next_idx & hermon_send_wq->num_wqes ) ? 1 : 0 );
01605         wqe_idx_mask = ( wq->num_wqes - 1 );
01606         if ( wq->iobufs[ wqe_idx & wqe_idx_mask ] ) {
01607                 DBGC ( hermon, "Hermon %p QPN %#lx send queue full",
01608                        hermon, qp->qpn );
01609                 return -ENOBUFS;
01610         }
01611         wq->iobufs[ wqe_idx & wqe_idx_mask ] = iobuf;
01612         wqe = &hermon_send_wq->wqe[wqe_idx];
01613 
01614         /* Construct work queue entry */
01615         memset ( ( ( ( void * ) wqe ) + 4 /* avoid ctrl.owner */ ), 0,
01616                    ( sizeof ( *wqe ) - 4 ) );
01617         assert ( qp->type < ( sizeof ( hermon_fill_send_wqe ) /
01618                               sizeof ( hermon_fill_send_wqe[0] ) ) );
01619         assert ( hermon_fill_send_wqe[qp->type] != NULL );
01620         opcode = hermon_fill_send_wqe[qp->type] ( ibdev, qp, dest, iobuf, wqe );
01621         barrier();
01622         MLX_FILL_2 ( &wqe->ctrl, 0,
01623                      opcode, opcode,
01624                      owner, owner );
01625         DBGCP ( hermon, "Hermon %p QPN %#lx posting send WQE %#lx:\n",
01626                 hermon, qp->qpn, wqe_idx );
01627         DBGCP_HDA ( hermon, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) );
01628 
01629         /* Ring doorbell register */
01630         MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn );
01631         barrier();
01632         writel ( db_reg.dword[0], hermon_send_wq->doorbell );
01633 
01634         /* Update work queue's index */
01635         wq->next_idx++;
01636 
01637         return 0;
01638 }
01639 
01640 /**
01641  * Post receive work queue entry
01642  *
01643  * @v ibdev             Infiniband device
01644  * @v qp                Queue pair
01645  * @v iobuf             I/O buffer
01646  * @ret rc              Return status code
01647  */
01648 static int hermon_post_recv ( struct ib_device *ibdev,
01649                               struct ib_queue_pair *qp,
01650                               struct io_buffer *iobuf ) {
01651         struct hermon *hermon = ib_get_drvdata ( ibdev );
01652         struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
01653         struct ib_work_queue *wq = &qp->recv;
01654         struct hermon_recv_work_queue *hermon_recv_wq = &hermon_qp->recv;
01655         struct hermonprm_recv_wqe *wqe;
01656         struct hermonprm_wqe_segment_data_ptr *data;
01657         struct ib_global_route_header *grh;
01658         unsigned int wqe_idx_mask;
01659 
01660         /* Allocate work queue entry */
01661         wqe_idx_mask = ( wq->num_wqes - 1 );
01662         if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
01663                 DBGC ( hermon, "Hermon %p QPN %#lx receive queue full",
01664                        hermon, qp->qpn );
01665                 return -ENOBUFS;
01666         }
01667         wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
01668         wqe = &hermon_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
01669 
01670         /* Construct work queue entry */
01671         data = &wqe->data[0];
01672         if ( hermon_qp->recv.grh ) {
01673                 grh = &hermon_qp->recv.grh[wq->next_idx & wqe_idx_mask];
01674                 MLX_FILL_1 ( data, 0, byte_count, sizeof ( *grh ) );
01675                 MLX_FILL_1 ( data, 1, l_key, hermon->lkey );
01676                 MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( grh ) );
01677                 MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( grh ) );
01678                 data++;
01679         }
01680         MLX_FILL_1 ( data, 0, byte_count, iob_tailroom ( iobuf ) );
01681         MLX_FILL_1 ( data, 1, l_key, hermon->lkey );
01682         MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( iobuf->data ) );
01683         MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( iobuf->data ) );
01684 
01685         /* Update work queue's index */
01686         wq->next_idx++;
01687 
01688         /* Update doorbell record */
01689         barrier();
01690         MLX_FILL_1 ( hermon_recv_wq->doorbell, 0, receive_wqe_counter,
01691                      ( wq->next_idx & 0xffff ) );
01692 
01693         return 0;
01694 }
01695 
01696 /**
01697  * Handle completion
01698  *
01699  * @v ibdev             Infiniband device
01700  * @v cq                Completion queue
01701  * @v cqe               Hardware completion queue entry
01702  * @ret rc              Return status code
01703  */
01704 static int hermon_complete ( struct ib_device *ibdev,
01705                              struct ib_completion_queue *cq,
01706                              union hermonprm_completion_entry *cqe ) {
01707         struct hermon *hermon = ib_get_drvdata ( ibdev );
01708         struct hermon_queue_pair *hermon_qp;
01709         struct ib_work_queue *wq;
01710         struct ib_queue_pair *qp;
01711         struct io_buffer *iobuf;
01712         struct ib_address_vector recv_dest;
01713         struct ib_address_vector recv_source;
01714         struct ib_global_route_header *grh;
01715         struct ib_address_vector *source;
01716         unsigned int opcode;
01717         unsigned long qpn;
01718         int is_send;
01719         unsigned long wqe_idx;
01720         unsigned long wqe_idx_mask;
01721         size_t len;
01722         int rc = 0;
01723 
01724         /* Parse completion */
01725         qpn = MLX_GET ( &cqe->normal, qpn );
01726         is_send = MLX_GET ( &cqe->normal, s_r );
01727         opcode = MLX_GET ( &cqe->normal, opcode );
01728         if ( opcode >= HERMON_OPCODE_RECV_ERROR ) {
01729                 /* "s" field is not valid for error opcodes */
01730                 is_send = ( opcode == HERMON_OPCODE_SEND_ERROR );
01731                 DBGC ( hermon, "Hermon %p CQN %#lx syndrome %x vendor %x\n",
01732                        hermon, cq->cqn, MLX_GET ( &cqe->error, syndrome ),
01733                        MLX_GET ( &cqe->error, vendor_error_syndrome ) );
01734                 rc = -EIO;
01735                 /* Don't return immediately; propagate error to completer */
01736         }
01737 
01738         /* Identify work queue */
01739         wq = ib_find_wq ( cq, qpn, is_send );
01740         if ( ! wq ) {
01741                 DBGC ( hermon, "Hermon %p CQN %#lx unknown %s QPN %#lx\n",
01742                        hermon, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
01743                 return -EIO;
01744         }
01745         qp = wq->qp;
01746         hermon_qp = ib_qp_get_drvdata ( qp );
01747 
01748         /* Identify work queue entry */
01749         wqe_idx = MLX_GET ( &cqe->normal, wqe_counter );
01750         wqe_idx_mask = ( wq->num_wqes - 1 );
01751         DBGCP ( hermon, "Hermon %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n",
01752                 hermon, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
01753                 wqe_idx );
01754         DBGCP_HDA ( hermon, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
01755 
01756         /* Identify I/O buffer */
01757         iobuf = wq->iobufs[ wqe_idx & wqe_idx_mask ];
01758         if ( ! iobuf ) {
01759                 DBGC ( hermon, "Hermon %p CQN %#lx QPN %#lx empty %s WQE "
01760                        "%#lx\n", hermon, cq->cqn, qp->qpn,
01761                        ( is_send ? "send" : "recv" ), wqe_idx );
01762                 return -EIO;
01763         }
01764         wq->iobufs[ wqe_idx & wqe_idx_mask ] = NULL;
01765 
01766         if ( is_send ) {
01767                 /* Hand off to completion handler */
01768                 ib_complete_send ( ibdev, qp, iobuf, rc );
01769         } else {
01770                 /* Set received length */
01771                 len = MLX_GET ( &cqe->normal, byte_cnt );
01772                 memset ( &recv_dest, 0, sizeof ( recv_dest ) );
01773                 recv_dest.qpn = qpn;
01774                 memset ( &recv_source, 0, sizeof ( recv_source ) );
01775                 switch ( qp->type ) {
01776                 case IB_QPT_SMI:
01777                 case IB_QPT_GSI:
01778                 case IB_QPT_UD:
01779                         /* Locate corresponding GRH */
01780                         assert ( hermon_qp->recv.grh != NULL );
01781                         grh = &hermon_qp->recv.grh[ wqe_idx & wqe_idx_mask ];
01782                         len -= sizeof ( *grh );
01783                         /* Construct address vector */
01784                         source = &recv_source;
01785                         source->qpn = MLX_GET ( &cqe->normal, srq_rqpn );
01786                         source->lid = MLX_GET ( &cqe->normal, slid_smac47_32 );
01787                         source->sl = MLX_GET ( &cqe->normal, sl );
01788                         recv_dest.gid_present = source->gid_present =
01789                                 MLX_GET ( &cqe->normal, g );
01790                         memcpy ( &recv_dest.gid, &grh->dgid,
01791                                  sizeof ( recv_dest.gid ) );
01792                         memcpy ( &source->gid, &grh->sgid,
01793                                  sizeof ( source->gid ) );
01794                         break;
01795                 case IB_QPT_RC:
01796                         source = &qp->av;
01797                         break;
01798                 case IB_QPT_ETH:
01799                         /* Construct address vector */
01800                         source = &recv_source;
01801                         source->vlan_present = MLX_GET ( &cqe->normal, vlan );
01802                         source->vlan = MLX_GET ( &cqe->normal, vid );
01803                         break;
01804                 default:
01805                         assert ( 0 );
01806                         return -EINVAL;
01807                 }
01808                 assert ( len <= iob_tailroom ( iobuf ) );
01809                 iob_put ( iobuf, len );
01810                 /* Hand off to completion handler */
01811                 ib_complete_recv ( ibdev, qp, &recv_dest, source, iobuf, rc );
01812         }
01813 
01814         return rc;
01815 }
01816 
01817 /**
01818  * Poll completion queue
01819  *
01820  * @v ibdev             Infiniband device
01821  * @v cq                Completion queue
01822  */
01823 static void hermon_poll_cq ( struct ib_device *ibdev,
01824                              struct ib_completion_queue *cq ) {
01825         struct hermon *hermon = ib_get_drvdata ( ibdev );
01826         struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
01827         union hermonprm_completion_entry *cqe;
01828         unsigned int cqe_idx_mask;
01829         int rc;
01830 
01831         while ( 1 ) {
01832                 /* Look for completion entry */
01833                 cqe_idx_mask = ( cq->num_cqes - 1 );
01834                 cqe = &hermon_cq->cqe[cq->next_idx & cqe_idx_mask];
01835                 if ( MLX_GET ( &cqe->normal, owner ) ^
01836                      ( ( cq->next_idx & cq->num_cqes ) ? 1 : 0 ) ) {
01837                         /* Entry still owned by hardware; end of poll */
01838                         break;
01839                 }
01840 
01841                 /* Handle completion */
01842                 if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) {
01843                         DBGC ( hermon, "Hermon %p CQN %#lx failed to complete:"
01844                                " %s\n", hermon, cq->cqn, strerror ( rc ) );
01845                         DBGC_HDA ( hermon, virt_to_phys ( cqe ),
01846                                    cqe, sizeof ( *cqe ) );
01847                 }
01848 
01849                 /* Update completion queue's index */
01850                 cq->next_idx++;
01851 
01852                 /* Update doorbell record */
01853                 MLX_FILL_1 ( hermon_cq->doorbell, 0, update_ci,
01854                              ( cq->next_idx & 0x00ffffffUL ) );
01855         }
01856 }
01857 
01858 /***************************************************************************
01859  *
01860  * Event queues
01861  *
01862  ***************************************************************************
01863  */
01864 
01865 /**
01866  * Create event queue
01867  *
01868  * @v hermon            Hermon device
01869  * @ret rc              Return status code
01870  */
01871 static int hermon_create_eq ( struct hermon *hermon ) {
01872         struct hermon_event_queue *hermon_eq = &hermon->eq;
01873         struct hermonprm_eqc eqctx;
01874         struct hermonprm_event_mask mask;
01875         unsigned int i;
01876         int rc;
01877 
01878         /* Select event queue number */
01879         hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars );
01880         if ( hermon_eq->eqn < hermon->cap.reserved_eqs )
01881                 hermon_eq->eqn = hermon->cap.reserved_eqs;
01882 
01883         /* Calculate doorbell address */
01884         hermon_eq->doorbell =
01885                 ( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) );
01886 
01887         /* Allocate event queue itself */
01888         hermon_eq->eqe_size =
01889                 ( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) );
01890         hermon_eq->eqe = malloc_dma ( hermon_eq->eqe_size,
01891                                       sizeof ( hermon_eq->eqe[0] ) );
01892         if ( ! hermon_eq->eqe ) {
01893                 rc = -ENOMEM;
01894                 goto err_eqe;
01895         }
01896         memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size );
01897         for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) {
01898                 MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 );
01899         }
01900         barrier();
01901 
01902         /* Allocate MTT entries */
01903         if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe,
01904                                        hermon_eq->eqe_size,
01905                                        &hermon_eq->mtt ) ) != 0 )
01906                 goto err_alloc_mtt;
01907 
01908         /* Hand queue over to hardware */
01909         memset ( &eqctx, 0, sizeof ( eqctx ) );
01910         MLX_FILL_2 ( &eqctx, 0,
01911                      st, 0xa /* "Fired" */,
01912                      oi, 1 );
01913         MLX_FILL_1 ( &eqctx, 2,
01914                      page_offset, ( hermon_eq->mtt.page_offset >> 5 ) );
01915         MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) );
01916         MLX_FILL_H ( &eqctx, 6, mtt_base_addr_h,
01917                      hermon_eq->mtt.mtt_base_addr );
01918         MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l,
01919                      ( hermon_eq->mtt.mtt_base_addr >> 3 ) );
01920         if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn,
01921                                           &eqctx ) ) != 0 ) {
01922                 DBGC ( hermon, "Hermon %p EQN %#lx SW2HW_EQ failed: %s\n",
01923                        hermon, hermon_eq->eqn, strerror ( rc ) );
01924                 goto err_sw2hw_eq;
01925         }
01926 
01927         /* Map all events to this event queue */
01928         memset ( &mask, 0xff, sizeof ( mask ) );
01929         if ( ( rc = hermon_cmd_map_eq ( hermon,
01930                                         ( HERMON_MAP_EQ | hermon_eq->eqn ),
01931                                         &mask ) ) != 0 ) {
01932                 DBGC ( hermon, "Hermon %p EQN %#lx MAP_EQ failed: %s\n",
01933                        hermon, hermon_eq->eqn, strerror ( rc )  );
01934                 goto err_map_eq;
01935         }
01936 
01937         DBGC ( hermon, "Hermon %p EQN %#lx ring [%08lx,%08lx), doorbell "
01938                "%08lx\n", hermon, hermon_eq->eqn,
01939                virt_to_phys ( hermon_eq->eqe ),
01940                ( virt_to_phys ( hermon_eq->eqe ) + hermon_eq->eqe_size ),
01941                virt_to_phys ( hermon_eq->doorbell ) );
01942         return 0;
01943 
01944  err_map_eq:
01945         hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx );
01946  err_sw2hw_eq:
01947         hermon_free_mtt ( hermon, &hermon_eq->mtt );
01948  err_alloc_mtt:
01949         free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
01950  err_eqe:
01951         memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
01952         return rc;
01953 }
01954 
01955 /**
01956  * Destroy event queue
01957  *
01958  * @v hermon            Hermon device
01959  */
01960 static void hermon_destroy_eq ( struct hermon *hermon ) {
01961         struct hermon_event_queue *hermon_eq = &hermon->eq;
01962         struct hermonprm_eqc eqctx;
01963         struct hermonprm_event_mask mask;
01964         int rc;
01965 
01966         /* Unmap events from event queue */
01967         memset ( &mask, 0xff, sizeof ( mask ) );
01968         if ( ( rc = hermon_cmd_map_eq ( hermon,
01969                                         ( HERMON_UNMAP_EQ | hermon_eq->eqn ),
01970                                         &mask ) ) != 0 ) {
01971                 DBGC ( hermon, "Hermon %p EQN %#lx FATAL MAP_EQ failed to "
01972                        "unmap: %s\n", hermon, hermon_eq->eqn, strerror ( rc ) );
01973                 /* Continue; HCA may die but system should survive */
01974         }
01975 
01976         /* Take ownership back from hardware */
01977         if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn,
01978                                           &eqctx ) ) != 0 ) {
01979                 DBGC ( hermon, "Hermon %p EQN %#lx FATAL HW2SW_EQ failed: %s\n",
01980                        hermon, hermon_eq->eqn, strerror ( rc ) );
01981                 /* Leak memory and return; at least we avoid corruption */
01982                 return;
01983         }
01984 
01985         /* Free MTT entries */
01986         hermon_free_mtt ( hermon, &hermon_eq->mtt );
01987 
01988         /* Free memory */
01989         free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
01990         memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
01991 }
01992 
01993 /**
01994  * Handle port state event
01995  *
01996  * @v hermon            Hermon device
01997  * @v eqe               Port state change event queue entry
01998  */
01999 static void hermon_event_port_state_change ( struct hermon *hermon,
02000                                              union hermonprm_event_entry *eqe){
02001         unsigned int port;
02002         int link_up;
02003 
02004         /* Get port and link status */
02005         port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
02006         link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
02007         DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ),
02008                ( link_up ? "up" : "down" ) );
02009 
02010         /* Sanity check */
02011         if ( port >= hermon->cap.num_ports ) {
02012                 DBGC ( hermon, "Hermon %p port %d does not exist!\n",
02013                        hermon, ( port + 1 ) );
02014                 return;
02015         }
02016 
02017         /* Notify device of port state change */
02018         hermon->port[port].type->state_change ( hermon, &hermon->port[port],
02019                                                 link_up );
02020 }
02021 
02022 /**
02023  * Poll event queue
02024  *
02025  * @v ibdev             Infiniband device
02026  */
02027 static void hermon_poll_eq ( struct ib_device *ibdev ) {
02028         struct hermon *hermon = ib_get_drvdata ( ibdev );
02029         struct hermon_event_queue *hermon_eq = &hermon->eq;
02030         union hermonprm_event_entry *eqe;
02031         union hermonprm_doorbell_register db_reg;
02032         unsigned int eqe_idx_mask;
02033         unsigned int event_type;
02034 
02035         /* No event is generated upon reaching INIT, so we must poll
02036          * separately for link state changes while we remain DOWN.
02037          */
02038         if ( ib_is_open ( ibdev ) &&
02039              ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) {
02040                 ib_smc_update ( ibdev, hermon_mad );
02041         }
02042 
02043         /* Poll event queue */
02044         while ( 1 ) {
02045                 /* Look for event entry */
02046                 eqe_idx_mask = ( HERMON_NUM_EQES - 1 );
02047                 eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask];
02048                 if ( MLX_GET ( &eqe->generic, owner ) ^
02049                      ( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) {
02050                         /* Entry still owned by hardware; end of poll */
02051                         break;
02052                 }
02053                 DBGCP ( hermon, "Hermon %p EQN %#lx event:\n",
02054                         hermon, hermon_eq->eqn );
02055                 DBGCP_HDA ( hermon, virt_to_phys ( eqe ),
02056                             eqe, sizeof ( *eqe ) );
02057 
02058                 /* Handle event */
02059                 event_type = MLX_GET ( &eqe->generic, event_type );
02060                 switch ( event_type ) {
02061                 case HERMON_EV_PORT_STATE_CHANGE:
02062                         hermon_event_port_state_change ( hermon, eqe );
02063                         break;
02064                 default:
02065                         DBGC ( hermon, "Hermon %p EQN %#lx unrecognised event "
02066                                "type %#x:\n",
02067                                hermon, hermon_eq->eqn, event_type );
02068                         DBGC_HDA ( hermon, virt_to_phys ( eqe ),
02069                                    eqe, sizeof ( *eqe ) );
02070                         break;
02071                 }
02072 
02073                 /* Update event queue's index */
02074                 hermon_eq->next_idx++;
02075 
02076                 /* Ring doorbell */
02077                 MLX_FILL_1 ( &db_reg.event, 0,
02078                              ci, ( hermon_eq->next_idx & 0x00ffffffUL ) );
02079                 writel ( db_reg.dword[0], hermon_eq->doorbell );
02080         }
02081 }
02082 
02083 /***************************************************************************
02084  *
02085  * Firmware control
02086  *
02087  ***************************************************************************
02088  */
02089 
02090 /**
02091  * Map virtual to physical address for firmware usage
02092  *
02093  * @v hermon            Hermon device
02094  * @v map               Mapping function
02095  * @v va                Virtual address
02096  * @v pa                Physical address
02097  * @v len               Length of region
02098  * @ret rc              Return status code
02099  */
02100 static int hermon_map_vpm ( struct hermon *hermon,
02101                             int ( *map ) ( struct hermon *hermon,
02102                             const struct hermonprm_virtual_physical_mapping* ),
02103                             uint64_t va, physaddr_t pa, size_t len ) {
02104         struct hermonprm_virtual_physical_mapping mapping;
02105         physaddr_t start;
02106         physaddr_t low;
02107         physaddr_t high;
02108         physaddr_t end;
02109         size_t size;
02110         int rc;
02111 
02112         /* Sanity checks */
02113         assert ( ( va & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
02114         assert ( ( pa & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
02115         assert ( ( len & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
02116         assert ( len != 0 );
02117 
02118         /* Calculate starting points */
02119         start = pa;
02120         end = ( start + len );
02121         size = ( 1UL << ( fls ( start ^ end ) - 1 ) );
02122         low = high = ( end & ~( size - 1 ) );
02123         assert ( start < low );
02124         assert ( high <= end );
02125 
02126         /* These mappings tend to generate huge volumes of
02127          * uninteresting debug data, which basically makes it
02128          * impossible to use debugging otherwise.
02129          */
02130         DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
02131 
02132         /* Map blocks in descending order of size */
02133         while ( size >= HERMON_PAGE_SIZE ) {
02134 
02135                 /* Find the next candidate block */
02136                 if ( ( low - size ) >= start ) {
02137                         low -= size;
02138                         pa = low;
02139                 } else if ( high <= ( end - size ) ) {
02140                         pa = high;
02141                         high += size;
02142                 } else {
02143                         size >>= 1;
02144                         continue;
02145                 }
02146                 assert ( ( va & ( size - 1 ) ) == 0 );
02147                 assert ( ( pa & ( size - 1 ) ) == 0 );
02148 
02149                 /* Map this block */
02150                 memset ( &mapping, 0, sizeof ( mapping ) );
02151                 MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) );
02152                 MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) );
02153                 MLX_FILL_H ( &mapping, 2, pa_h, pa );
02154                 MLX_FILL_2 ( &mapping, 3,
02155                              log2size, ( ( fls ( size ) - 1 ) - 12 ),
02156                              pa_l, ( pa >> 12 ) );
02157                 if ( ( rc = map ( hermon, &mapping ) ) != 0 ) {
02158                         DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
02159                         DBGC ( hermon, "Hermon %p could not map %08llx+%zx to "
02160                                "%08lx: %s\n",
02161                                hermon, va, size, pa, strerror ( rc ) );
02162                         return rc;
02163                 }
02164                 va += size;
02165         }
02166         assert ( low == start );
02167         assert ( high == end );
02168 
02169         DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
02170         return 0;
02171 }
02172 
02173 /**
02174  * Start firmware running
02175  *
02176  * @v hermon            Hermon device
02177  * @ret rc              Return status code
02178  */
02179 static int hermon_start_firmware ( struct hermon *hermon ) {
02180         struct hermonprm_query_fw fw;
02181         unsigned int fw_pages;
02182         size_t fw_len;
02183         physaddr_t fw_base;
02184         int rc;
02185 
02186         /* Get firmware parameters */
02187         if ( ( rc = hermon_cmd_query_fw ( hermon, &fw ) ) != 0 ) {
02188                 DBGC ( hermon, "Hermon %p could not query firmware: %s\n",
02189                        hermon, strerror ( rc ) );
02190                 goto err_query_fw;
02191         }
02192         DBGC ( hermon, "Hermon %p firmware version %d.%d.%d\n", hermon,
02193                MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
02194                MLX_GET ( &fw, fw_rev_subminor ) );
02195         fw_pages = MLX_GET ( &fw, fw_pages );
02196         DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n",
02197                hermon, fw_pages, ( fw_pages * 4 ) );
02198 
02199         /* Allocate firmware pages and map firmware area */
02200         fw_len = ( fw_pages * HERMON_PAGE_SIZE );
02201         if ( ! hermon->firmware_area ) {
02202                 hermon->firmware_len = fw_len;
02203                 hermon->firmware_area = umalloc ( hermon->firmware_len );
02204                 if ( ! hermon->firmware_area ) {
02205                         rc = -ENOMEM;
02206                         goto err_alloc_fa;
02207                 }
02208         } else {
02209                 assert ( hermon->firmware_len == fw_len );
02210         }
02211         fw_base = user_to_phys ( hermon->firmware_area, 0 );
02212         DBGC ( hermon, "Hermon %p firmware area at physical [%08lx,%08lx)\n",
02213                hermon, fw_base, ( fw_base + fw_len ) );
02214         if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_fa,
02215                                      0, fw_base, fw_len ) ) != 0 ) {
02216                 DBGC ( hermon, "Hermon %p could not map firmware: %s\n",
02217                        hermon, strerror ( rc ) );
02218                 goto err_map_fa;
02219         }
02220 
02221         /* Start firmware */
02222         if ( ( rc = hermon_cmd_run_fw ( hermon ) ) != 0 ) {
02223                 DBGC ( hermon, "Hermon %p could not run firmware: %s\n",
02224                        hermon, strerror ( rc ) );
02225                 goto err_run_fw;
02226         }
02227 
02228         DBGC ( hermon, "Hermon %p firmware started\n", hermon );
02229         return 0;
02230 
02231  err_run_fw:
02232  err_map_fa:
02233         hermon_cmd_unmap_fa ( hermon );
02234  err_alloc_fa:
02235  err_query_fw:
02236         return rc;
02237 }
02238 
02239 /**
02240  * Stop firmware running
02241  *
02242  * @v hermon            Hermon device
02243  */
02244 static void hermon_stop_firmware ( struct hermon *hermon ) {
02245         int rc;
02246 
02247         if ( ( rc = hermon_cmd_unmap_fa ( hermon ) ) != 0 ) {
02248                 DBGC ( hermon, "Hermon %p FATAL could not stop firmware: %s\n",
02249                        hermon, strerror ( rc ) );
02250                 /* Leak memory and return; at least we avoid corruption */
02251                 hermon->firmware_area = UNULL;
02252                 return;
02253         }
02254 }
02255 
02256 /***************************************************************************
02257  *
02258  * Infinihost Context Memory management
02259  *
02260  ***************************************************************************
02261  */
02262 
02263 /**
02264  * Get device limits
02265  *
02266  * @v hermon            Hermon device
02267  * @ret rc              Return status code
02268  */
02269 static int hermon_get_cap ( struct hermon *hermon ) {
02270         struct hermonprm_query_dev_cap dev_cap;
02271         int rc;
02272 
02273         if ( ( rc = hermon_cmd_query_dev_cap ( hermon, &dev_cap ) ) != 0 ) {
02274                 DBGC ( hermon, "Hermon %p could not get device limits: %s\n",
02275                        hermon, strerror ( rc ) );
02276                 return rc;
02277         }
02278 
02279         hermon->cap.cmpt_entry_size = MLX_GET ( &dev_cap, c_mpt_entry_sz );
02280         hermon->cap.reserved_qps =
02281                 ( 1 << MLX_GET ( &dev_cap, log2_rsvd_qps ) );
02282         hermon->cap.qpc_entry_size = MLX_GET ( &dev_cap, qpc_entry_sz );
02283         hermon->cap.altc_entry_size = MLX_GET ( &dev_cap, altc_entry_sz );
02284         hermon->cap.auxc_entry_size = MLX_GET ( &dev_cap, aux_entry_sz );
02285         hermon->cap.reserved_srqs =
02286                 ( 1 << MLX_GET ( &dev_cap, log2_rsvd_srqs ) );
02287         hermon->cap.srqc_entry_size = MLX_GET ( &dev_cap, srq_entry_sz );
02288         hermon->cap.reserved_cqs =
02289                 ( 1 << MLX_GET ( &dev_cap, log2_rsvd_cqs ) );
02290         hermon->cap.cqc_entry_size = MLX_GET ( &dev_cap, cqc_entry_sz );
02291         hermon->cap.reserved_eqs = MLX_GET ( &dev_cap, num_rsvd_eqs );
02292         if ( hermon->cap.reserved_eqs == 0 ) {
02293                 /* Backward compatibility */
02294                 hermon->cap.reserved_eqs =
02295                         ( 1 << MLX_GET ( &dev_cap, log2_rsvd_eqs ) );
02296         }
02297         hermon->cap.eqc_entry_size = MLX_GET ( &dev_cap, eqc_entry_sz );
02298         hermon->cap.reserved_mtts =
02299                 ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mtts ) );
02300         hermon->cap.mtt_entry_size = MLX_GET ( &dev_cap, mtt_entry_sz );
02301         hermon->cap.reserved_mrws =
02302                 ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
02303         hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
02304         hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
02305         hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
02306         hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );
02307 
02308         /* Sanity check */
02309         if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
02310                 DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
02311                        hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
02312                 hermon->cap.num_ports = HERMON_MAX_PORTS;
02313         }
02314 
02315         return 0;
02316 }
02317 
02318 /**
02319  * Align ICM table
02320  *
02321  * @v icm_offset        Current ICM offset
02322  * @v len               ICM table length
02323  * @ret icm_offset      ICM offset
02324  */
02325 static uint64_t icm_align ( uint64_t icm_offset, size_t len ) {
02326 
02327         /* Round up to a multiple of the table size */
02328         assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
02329         return ( ( icm_offset + len - 1 ) & ~( ( ( uint64_t ) len ) - 1 ) );
02330 }
02331 
02332 /**
02333  * Map ICM (allocating if necessary)
02334  *
02335  * @v hermon            Hermon device
02336  * @v init_hca          INIT_HCA structure to fill in
02337  * @ret rc              Return status code
02338  */
02339 static int hermon_map_icm ( struct hermon *hermon,
02340                             struct hermonprm_init_hca *init_hca ) {
02341         struct hermonprm_scalar_parameter icm_size;
02342         struct hermonprm_scalar_parameter icm_aux_size;
02343         uint64_t icm_offset = 0;
02344         unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs;
02345         unsigned int log_num_mtts, log_num_mpts, log_num_mcs;
02346         size_t cmpt_max_len;
02347         size_t icm_len, icm_aux_len;
02348         size_t len;
02349         physaddr_t icm_phys;
02350         int i;
02351         int rc;
02352 
02353         /*
02354          * Start by carving up the ICM virtual address space
02355          *
02356          */
02357 
02358         /* Calculate number of each object type within ICM */
02359         log_num_qps = fls ( hermon->cap.reserved_qps +
02360                             HERMON_RSVD_SPECIAL_QPS + HERMON_MAX_QPS - 1 );
02361         log_num_srqs = fls ( hermon->cap.reserved_srqs - 1 );
02362         log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 );
02363         log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 );
02364         log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 );
02365         log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 );
02366         log_num_mcs = HERMON_LOG_MULTICAST_HASH_SIZE;
02367 
02368         /* ICM starts with the cMPT tables, which are sparse */
02369         cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES *
02370                          ( ( uint64_t ) hermon->cap.cmpt_entry_size ) );
02371         len = ( ( ( ( 1 << log_num_qps ) * hermon->cap.cmpt_entry_size ) +
02372                   HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
02373         hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset;
02374         hermon->icm_map[HERMON_ICM_QP_CMPT].len = len;
02375         icm_offset += cmpt_max_len;
02376         len = ( ( ( ( 1 << log_num_srqs ) * hermon->cap.cmpt_entry_size ) +
02377                   HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
02378         hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset;
02379         hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = len;
02380         icm_offset += cmpt_max_len;
02381         len = ( ( ( ( 1 << log_num_cqs ) * hermon->cap.cmpt_entry_size ) +
02382                   HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
02383         hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset;
02384         hermon->icm_map[HERMON_ICM_CQ_CMPT].len = len;
02385         icm_offset += cmpt_max_len;
02386         len = ( ( ( ( 1 << log_num_eqs ) * hermon->cap.cmpt_entry_size ) +
02387                   HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
02388         hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset;
02389         hermon->icm_map[HERMON_ICM_EQ_CMPT].len = len;
02390         icm_offset += cmpt_max_len;
02391 
02392         hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset;
02393 
02394         /* Queue pair contexts */
02395         len = ( ( 1 << log_num_qps ) * hermon->cap.qpc_entry_size );
02396         icm_offset = icm_align ( icm_offset, len );
02397         MLX_FILL_1 ( init_hca, 12,
02398                      qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h,
02399                      ( icm_offset >> 32 ) );
02400         MLX_FILL_2 ( init_hca, 13,
02401                      qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l,
02402                      ( icm_offset >> 5 ),
02403                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
02404                      log_num_qps );
02405         DBGC ( hermon, "Hermon %p ICM QPC is %d x %#zx at [%08llx,%08llx)\n",
02406                hermon, ( 1 << log_num_qps ), hermon->cap.qpc_entry_size,
02407                icm_offset, ( icm_offset + len ) );
02408         icm_offset += len;
02409 
02410         /* Extended alternate path contexts */
02411         len = ( ( 1 << log_num_qps ) * hermon->cap.altc_entry_size );
02412         icm_offset = icm_align ( icm_offset, len );
02413         MLX_FILL_1 ( init_hca, 24,
02414                      qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h,
02415                      ( icm_offset >> 32 ) );
02416         MLX_FILL_1 ( init_hca, 25,
02417                      qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l,
02418                      icm_offset );
02419         DBGC ( hermon, "Hermon %p ICM ALTC is %d x %#zx at [%08llx,%08llx)\n",
02420                hermon, ( 1 << log_num_qps ), hermon->cap.altc_entry_size,
02421                icm_offset, ( icm_offset + len ) );
02422         icm_offset += len;
02423 
02424         /* Extended auxiliary contexts */
02425         len = ( ( 1 << log_num_qps ) * hermon->cap.auxc_entry_size );
02426         icm_offset = icm_align ( icm_offset, len );
02427         MLX_FILL_1 ( init_hca, 28,
02428                      qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h,
02429                      ( icm_offset >> 32 ) );
02430         MLX_FILL_1 ( init_hca, 29,
02431                      qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l,
02432                      icm_offset );
02433         DBGC ( hermon, "Hermon %p ICM AUXC is %d x %#zx at [%08llx,%08llx)\n",
02434                hermon, ( 1 << log_num_qps ), hermon->cap.auxc_entry_size,
02435                icm_offset, ( icm_offset + len ) );
02436         icm_offset += len;
02437 
02438         /* Shared receive queue contexts */
02439         len = ( ( 1 << log_num_srqs ) * hermon->cap.srqc_entry_size );
02440         icm_offset = icm_align ( icm_offset, len );
02441         MLX_FILL_1 ( init_hca, 18,
02442                      qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h,
02443                      ( icm_offset >> 32 ) );
02444         MLX_FILL_2 ( init_hca, 19,
02445                      qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l,
02446                      ( icm_offset >> 5 ),
02447                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
02448                      log_num_srqs );
02449         DBGC ( hermon, "Hermon %p ICM SRQC is %d x %#zx at [%08llx,%08llx)\n",
02450                hermon, ( 1 << log_num_srqs ), hermon->cap.srqc_entry_size,
02451                icm_offset, ( icm_offset + len ) );
02452         icm_offset += len;
02453 
02454         /* Completion queue contexts */
02455         len = ( ( 1 << log_num_cqs ) * hermon->cap.cqc_entry_size );
02456         icm_offset = icm_align ( icm_offset, len );
02457         MLX_FILL_1 ( init_hca, 20,
02458                      qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h,
02459                      ( icm_offset >> 32 ) );
02460         MLX_FILL_2 ( init_hca, 21,
02461                      qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l,
02462                      ( icm_offset >> 5 ),
02463                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
02464                      log_num_cqs );
02465         DBGC ( hermon, "Hermon %p ICM CQC is %d x %#zx at [%08llx,%08llx)\n",
02466                hermon, ( 1 << log_num_cqs ), hermon->cap.cqc_entry_size,
02467                icm_offset, ( icm_offset + len ) );
02468         icm_offset += len;
02469 
02470         /* Event queue contexts */
02471         len = ( ( 1 << log_num_eqs ) * hermon->cap.eqc_entry_size );
02472         icm_offset = icm_align ( icm_offset, len );
02473         MLX_FILL_1 ( init_hca, 32,
02474                      qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h,
02475                      ( icm_offset >> 32 ) );
02476         MLX_FILL_2 ( init_hca, 33,
02477                      qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l,
02478                      ( icm_offset >> 5 ),
02479                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq,
02480                      log_num_eqs );
02481         DBGC ( hermon, "Hermon %p ICM EQC is %d x %#zx at [%08llx,%08llx)\n",
02482                hermon, ( 1 << log_num_eqs ), hermon->cap.eqc_entry_size,
02483                icm_offset, ( icm_offset + len ) );
02484         icm_offset += len;
02485 
02486         /* Memory translation table */
02487         len = ( ( 1 << log_num_mtts ) * hermon->cap.mtt_entry_size );
02488         icm_offset = icm_align ( icm_offset, len );
02489         MLX_FILL_1 ( init_hca, 64,
02490                      tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) );
02491         MLX_FILL_1 ( init_hca, 65,
02492                      tpt_parameters.mtt_base_addr_l, icm_offset );
02493         DBGC ( hermon, "Hermon %p ICM MTT is %d x %#zx at [%08llx,%08llx)\n",
02494                hermon, ( 1 << log_num_mtts ), hermon->cap.mtt_entry_size,
02495                icm_offset, ( icm_offset + len ) );
02496         icm_offset += len;
02497 
02498         /* Memory protection table */
02499         len = ( ( 1 << log_num_mpts ) * hermon->cap.dmpt_entry_size );
02500         icm_offset = icm_align ( icm_offset, len );
02501         MLX_FILL_1 ( init_hca, 60,
02502                      tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) );
02503         MLX_FILL_1 ( init_hca, 61,
02504                      tpt_parameters.dmpt_base_adr_l, icm_offset );
02505         MLX_FILL_1 ( init_hca, 62,
02506                      tpt_parameters.log_dmpt_sz, log_num_mpts );
02507         DBGC ( hermon, "Hermon %p ICM DMPT is %d x %#zx at [%08llx,%08llx)\n",
02508                hermon, ( 1 << log_num_mpts ), hermon->cap.dmpt_entry_size,
02509                icm_offset, ( icm_offset + len ) );
02510         icm_offset += len;
02511 
02512         /* Multicast table */
02513         len = ( ( 1 << log_num_mcs ) * sizeof ( struct hermonprm_mcg_entry ) );
02514         icm_offset = icm_align ( icm_offset, len );
02515         MLX_FILL_1 ( init_hca, 48,
02516                      multicast_parameters.mc_base_addr_h,
02517                      ( icm_offset >> 32 ) );
02518         MLX_FILL_1 ( init_hca, 49,
02519                      multicast_parameters.mc_base_addr_l, icm_offset );
02520         MLX_FILL_1 ( init_hca, 52,
02521                      multicast_parameters.log_mc_table_entry_sz,
02522                      fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) );
02523         MLX_FILL_1 ( init_hca, 53,
02524                      multicast_parameters.log_mc_table_hash_sz, log_num_mcs );
02525         MLX_FILL_1 ( init_hca, 54,
02526                      multicast_parameters.log_mc_table_sz, log_num_mcs );
02527         DBGC ( hermon, "Hermon %p ICM MC is %d x %#zx at [%08llx,%08llx)\n",
02528                hermon, ( 1 << log_num_mcs ),
02529                sizeof ( struct hermonprm_mcg_entry ),
02530                icm_offset, ( icm_offset + len ) );
02531         icm_offset += len;
02532 
02533 
02534         hermon->icm_map[HERMON_ICM_OTHER].len =
02535                 ( icm_offset - hermon->icm_map[HERMON_ICM_OTHER].offset );
02536 
02537         /*
02538          * Allocate and map physical memory for (portions of) ICM
02539          *
02540          * Map is:
02541          *   ICM AUX area (aligned to its own size)
02542          *   cMPT areas
02543          *   Other areas
02544          */
02545 
02546         /* Calculate physical memory required for ICM */
02547         icm_len = 0;
02548         for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
02549                 icm_len += hermon->icm_map[i].len;
02550         }
02551 
02552         /* Get ICM auxiliary area size */
02553         memset ( &icm_size, 0, sizeof ( icm_size ) );
02554         MLX_FILL_1 ( &icm_size, 0, value_hi, ( icm_offset >> 32 ) );
02555         MLX_FILL_1 ( &icm_size, 1, value, icm_offset );
02556         if ( ( rc = hermon_cmd_set_icm_size ( hermon, &icm_size,
02557                                               &icm_aux_size ) ) != 0 ) {
02558                 DBGC ( hermon, "Hermon %p could not set ICM size: %s\n",
02559                        hermon, strerror ( rc ) );
02560                 goto err_set_icm_size;
02561         }
02562         icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * HERMON_PAGE_SIZE );
02563 
02564         /* Allocate ICM data and auxiliary area */
02565         DBGC ( hermon, "Hermon %p requires %zd kB ICM and %zd kB AUX ICM\n",
02566                hermon, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
02567         if ( ! hermon->icm ) {
02568                 hermon->icm_len = icm_len;
02569                 hermon->icm_aux_len = icm_aux_len;
02570                 hermon->icm = umalloc ( hermon->icm_aux_len + hermon->icm_len );
02571                 if ( ! hermon->icm ) {
02572                         rc = -ENOMEM;
02573                         goto err_alloc;
02574                 }
02575         } else {
02576                 assert ( hermon->icm_len == icm_len );
02577                 assert ( hermon->icm_aux_len == icm_aux_len );
02578         }
02579         icm_phys = user_to_phys ( hermon->icm, 0 );
02580 
02581         /* Map ICM auxiliary area */
02582         DBGC ( hermon, "Hermon %p mapping ICM AUX => %08lx\n",
02583                hermon, icm_phys );
02584         if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm_aux,
02585                                      0, icm_phys, icm_aux_len ) ) != 0 ) {
02586                 DBGC ( hermon, "Hermon %p could not map AUX ICM: %s\n",
02587                        hermon, strerror ( rc ) );
02588                 goto err_map_icm_aux;
02589         }
02590         icm_phys += icm_aux_len;
02591 
02592         /* MAP ICM area */
02593         for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
02594                 DBGC ( hermon, "Hermon %p mapping ICM %llx+%zx => %08lx\n",
02595                        hermon, hermon->icm_map[i].offset,
02596                        hermon->icm_map[i].len, icm_phys );
02597                 if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm,
02598                                              hermon->icm_map[i].offset,
02599                                              icm_phys,
02600                                              hermon->icm_map[i].len ) ) != 0 ){
02601                         DBGC ( hermon, "Hermon %p could not map ICM: %s\n",
02602                                hermon, strerror ( rc ) );
02603                         goto err_map_icm;
02604                 }
02605                 icm_phys += hermon->icm_map[i].len;
02606         }
02607 
02608         return 0;
02609 
02610  err_map_icm:
02611         assert ( i == 0 ); /* We don't handle partial failure at present */
02612  err_map_icm_aux:
02613         hermon_cmd_unmap_icm_aux ( hermon );
02614  err_alloc:
02615  err_set_icm_size:
02616         return rc;
02617 }
02618 
02619 /**
02620  * Unmap ICM
02621  *
02622  * @v hermon            Hermon device
02623  */
02624 static void hermon_unmap_icm ( struct hermon *hermon ) {
02625         struct hermonprm_scalar_parameter unmap_icm;
02626         int i;
02627 
02628         for ( i = ( HERMON_ICM_NUM_REGIONS - 1 ) ; i >= 0 ; i-- ) {
02629                 memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
02630                 MLX_FILL_1 ( &unmap_icm, 0, value_hi,
02631                              ( hermon->icm_map[i].offset >> 32 ) );
02632                 MLX_FILL_1 ( &unmap_icm, 1, value,
02633                              hermon->icm_map[i].offset );
02634                 hermon_cmd_unmap_icm ( hermon,
02635                                        ( 1 << fls ( ( hermon->icm_map[i].len /
02636                                                       HERMON_PAGE_SIZE ) - 1)),
02637                                        &unmap_icm );
02638         }
02639         hermon_cmd_unmap_icm_aux ( hermon );
02640 }
02641 
02642 /***************************************************************************
02643  *
02644  * Initialisation and teardown
02645  *
02646  ***************************************************************************
02647  */
02648 
02649 /**
02650  * Reset device
02651  *
02652  * @v hermon            Hermon device
02653  */
02654 static void hermon_reset ( struct hermon *hermon ) {
02655         struct pci_device *pci = hermon->pci;
02656         struct pci_config_backup backup;
02657         static const uint8_t backup_exclude[] =
02658                 PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
02659 
02660         /* Perform device reset and preserve PCI configuration */
02661         pci_backup ( pci, &backup, backup_exclude );
02662         writel ( HERMON_RESET_MAGIC,
02663                  ( hermon->config + HERMON_RESET_OFFSET ) );
02664         mdelay ( HERMON_RESET_WAIT_TIME_MS );
02665         pci_restore ( pci, &backup, backup_exclude );
02666 
02667         /* Reset command interface toggle */
02668         hermon->toggle = 0;
02669 }
02670 
02671 /**
02672  * Set up memory protection table
02673  *
02674  * @v hermon            Hermon device
02675  * @ret rc              Return status code
02676  */
02677 static int hermon_setup_mpt ( struct hermon *hermon ) {
02678         struct hermonprm_mpt mpt;
02679         uint32_t key;
02680         int rc;
02681 
02682         /* Derive key */
02683         key = ( hermon->cap.reserved_mrws | HERMON_MKEY_PREFIX );
02684         hermon->lkey = ( ( key << 8 ) | ( key >> 24 ) );
02685 
02686         /* Initialise memory protection table */
02687         memset ( &mpt, 0, sizeof ( mpt ) );
02688         MLX_FILL_7 ( &mpt, 0,
02689                      atomic, 1,
02690                      rw, 1,
02691                      rr, 1,
02692                      lw, 1,
02693                      lr, 1,
02694                      pa, 1,
02695                      r_w, 1 );
02696         MLX_FILL_1 ( &mpt, 2, mem_key, key );
02697         MLX_FILL_1 ( &mpt, 3,
02698                      pd, HERMON_GLOBAL_PD );
02699         MLX_FILL_1 ( &mpt, 10, len64, 1 );
02700         if ( ( rc = hermon_cmd_sw2hw_mpt ( hermon,
02701                                            hermon->cap.reserved_mrws,
02702                                            &mpt ) ) != 0 ) {
02703                 DBGC ( hermon, "Hermon %p could not set up MPT: %s\n",
02704                        hermon, strerror ( rc ) );
02705                 return rc;
02706         }
02707 
02708         return 0;
02709 }
02710 
02711 /**
02712  * Configure special queue pairs
02713  *
02714  * @v hermon            Hermon device
02715  * @ret rc              Return status code
02716  */
02717 static int hermon_configure_special_qps ( struct hermon *hermon ) {
02718         int rc;
02719 
02720         /* Special QP block must be aligned on its own size */
02721         hermon->special_qpn_base = ( ( hermon->cap.reserved_qps +
02722                                        HERMON_NUM_SPECIAL_QPS - 1 )
02723                                      & ~( HERMON_NUM_SPECIAL_QPS - 1 ) );
02724         hermon->qpn_base = ( hermon->special_qpn_base +
02725                              HERMON_NUM_SPECIAL_QPS );
02726         DBGC ( hermon, "Hermon %p special QPs at [%lx,%lx]\n", hermon,
02727                hermon->special_qpn_base, ( hermon->qpn_base - 1 ) );
02728 
02729         /* Issue command to configure special QPs */
02730         if ( ( rc = hermon_cmd_conf_special_qp ( hermon, 0x00,
02731                                           hermon->special_qpn_base ) ) != 0 ) {
02732                 DBGC ( hermon, "Hermon %p could not configure special QPs: "
02733                        "%s\n", hermon, strerror ( rc ) );
02734                 return rc;
02735         }
02736 
02737         return 0;
02738 }
02739 
02740 /**
02741  * Start Hermon device
02742  *
02743  * @v hermon            Hermon device
02744  * @v running           Firmware is already running
02745  * @ret rc              Return status code
02746  */
02747 static int hermon_start ( struct hermon *hermon, int running ) {
02748         struct hermonprm_init_hca init_hca;
02749         unsigned int i;
02750         int rc;
02751 
02752         /* Start firmware if not already running */
02753         if ( ! running ) {
02754                 if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 )
02755                         goto err_start_firmware;
02756         }
02757 
02758         /* Allocate and map ICM */
02759         memset ( &init_hca, 0, sizeof ( init_hca ) );
02760         if ( ( rc = hermon_map_icm ( hermon, &init_hca ) ) != 0 )
02761                 goto err_map_icm;
02762 
02763         /* Initialise HCA */
02764         MLX_FILL_1 ( &init_hca, 0, version, 0x02 /* "Must be 0x02" */ );
02765         MLX_FILL_1 ( &init_hca, 5, udp, 1 );
02766         MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 8 );
02767         if ( ( rc = hermon_cmd_init_hca ( hermon, &init_hca ) ) != 0 ) {
02768                 DBGC ( hermon, "Hermon %p could not initialise HCA: %s\n",
02769                        hermon, strerror ( rc ) );
02770                 goto err_init_hca;
02771         }
02772 
02773         /* Set up memory protection */
02774         if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
02775                 goto err_setup_mpt;
02776         for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
02777                 hermon->port[i].ibdev->rdma_key = hermon->lkey;
02778 
02779         /* Set up event queue */
02780         if ( ( rc = hermon_create_eq ( hermon ) ) != 0 )
02781                 goto err_create_eq;
02782 
02783         /* Configure special QPs */
02784         if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
02785                 goto err_conf_special_qps;
02786 
02787         return 0;
02788 
02789  err_conf_special_qps:
02790         hermon_destroy_eq ( hermon );
02791  err_create_eq:
02792  err_setup_mpt:
02793         hermon_cmd_close_hca ( hermon );
02794  err_init_hca:
02795         hermon_unmap_icm ( hermon );
02796  err_map_icm:
02797         hermon_stop_firmware ( hermon );
02798  err_start_firmware:
02799         return rc;
02800 }
02801 
02802 /**
02803  * Stop Hermon device
02804  *
02805  * @v hermon            Hermon device
02806  */
02807 static void hermon_stop ( struct hermon *hermon ) {
02808         hermon_destroy_eq ( hermon );
02809         hermon_cmd_close_hca ( hermon );
02810         hermon_unmap_icm ( hermon );
02811         hermon_stop_firmware ( hermon );
02812         hermon_reset ( hermon );
02813 }
02814 
02815 /**
02816  * Open Hermon device
02817  *
02818  * @v hermon            Hermon device
02819  * @ret rc              Return status code
02820  */
02821 static int hermon_open ( struct hermon *hermon ) {
02822         int rc;
02823 
02824         /* Start device if applicable */
02825         if ( hermon->open_count == 0 ) {
02826                 if ( ( rc = hermon_start ( hermon, 0 ) ) != 0 )
02827                         return rc;
02828         }
02829 
02830         /* Increment open counter */
02831         hermon->open_count++;
02832 
02833         return 0;
02834 }
02835 
02836 /**
02837  * Close Hermon device
02838  *
02839  * @v hermon            Hermon device
02840  */
02841 static void hermon_close ( struct hermon *hermon ) {
02842 
02843         /* Decrement open counter */
02844         assert ( hermon->open_count != 0 );
02845         hermon->open_count--;
02846 
02847         /* Stop device if applicable */
02848         if ( hermon->open_count == 0 )
02849                 hermon_stop ( hermon );
02850 }
02851 
02852 /***************************************************************************
02853  *
02854  * Infiniband link-layer operations
02855  *
02856  ***************************************************************************
02857  */
02858 
02859 /**
02860  * Initialise Infiniband link
02861  *
02862  * @v ibdev             Infiniband device
02863  * @ret rc              Return status code
02864  */
02865 static int hermon_ib_open ( struct ib_device *ibdev ) {
02866         struct hermon *hermon = ib_get_drvdata ( ibdev );
02867         union hermonprm_set_port set_port;
02868         int rc;
02869 
02870         /* Open hardware */
02871         if ( ( rc = hermon_open ( hermon ) ) != 0 )
02872                 goto err_open;
02873 
02874         /* Set port parameters */
02875         memset ( &set_port, 0, sizeof ( set_port ) );
02876         MLX_FILL_8 ( &set_port.ib, 0,
02877                      mmc, 1,
02878                      mvc, 1,
02879                      mp, 1,
02880                      mg, 1,
02881                      mtu_cap, IB_MTU_2048,
02882                      vl_cap, IB_VL_0,
02883                      rcm, 1,
02884                      lss, 1 );
02885         MLX_FILL_2 ( &set_port.ib, 10,
02886                      max_pkey, 1,
02887                      max_gid, 1 );
02888         MLX_FILL_1 ( &set_port.ib, 28,
02889                      link_speed_supported, 1 );
02890         if ( ( rc = hermon_cmd_set_port ( hermon, 0, ibdev->port,
02891                                           &set_port ) ) != 0 ) {
02892                 DBGC ( hermon, "Hermon %p port %d could not set port: %s\n",
02893                        hermon, ibdev->port, strerror ( rc ) );
02894                 goto err_set_port;
02895         }
02896 
02897         /* Initialise port */
02898         if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
02899                 DBGC ( hermon, "Hermon %p port %d could not initialise port: "
02900                        "%s\n", hermon, ibdev->port, strerror ( rc ) );
02901                 goto err_init_port;
02902         }
02903 
02904         /* Update MAD parameters */
02905         ib_smc_update ( ibdev, hermon_mad );
02906 
02907         return 0;
02908 
02909  err_init_port:
02910  err_set_port:
02911         hermon_close ( hermon );
02912  err_open:
02913         return rc;
02914 }
02915 
02916 /**
02917  * Close Infiniband link
02918  *
02919  * @v ibdev             Infiniband device
02920  */
02921 static void hermon_ib_close ( struct ib_device *ibdev ) {
02922         struct hermon *hermon = ib_get_drvdata ( ibdev );
02923         int rc;
02924 
02925         /* Close port */
02926         if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
02927                 DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
02928                        hermon, ibdev->port, strerror ( rc ) );
02929                 /* Nothing we can do about this */
02930         }
02931 
02932         /* Close hardware */
02933         hermon_close ( hermon );
02934 }
02935 
02936 /**
02937  * Inform embedded subnet management agent of a received MAD
02938  *
02939  * @v ibdev             Infiniband device
02940  * @v mad               MAD
02941  * @ret rc              Return status code
02942  */
02943 static int hermon_inform_sma ( struct ib_device *ibdev,
02944                                union ib_mad *mad ) {
02945         int rc;
02946 
02947         /* Send the MAD to the embedded SMA */
02948         if ( ( rc = hermon_mad ( ibdev, mad ) ) != 0 )
02949                 return rc;
02950 
02951         /* Update parameters held in software */
02952         ib_smc_update ( ibdev, hermon_mad );
02953 
02954         return 0;
02955 }
02956 
02957 /***************************************************************************
02958  *
02959  * Multicast group operations
02960  *
02961  ***************************************************************************
02962  */
02963 
02964 /**
02965  * Attach to multicast group
02966  *
02967  * @v ibdev             Infiniband device
02968  * @v qp                Queue pair
02969  * @v gid               Multicast GID
02970  * @ret rc              Return status code
02971  */
02972 static int hermon_mcast_attach ( struct ib_device *ibdev,
02973                                  struct ib_queue_pair *qp,
02974                                  union ib_gid *gid ) {
02975         struct hermon *hermon = ib_get_drvdata ( ibdev );
02976         struct hermonprm_mgm_hash hash;
02977         struct hermonprm_mcg_entry mcg;
02978         unsigned int index;
02979         int rc;
02980 
02981         /* Generate hash table index */
02982         if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
02983                 DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
02984                        hermon, strerror ( rc ) );
02985                 return rc;
02986         }
02987         index = MLX_GET ( &hash, hash );
02988 
02989         /* Check for existing hash table entry */
02990         if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) {
02991                 DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n",
02992                        hermon, index, strerror ( rc ) );
02993                 return rc;
02994         }
02995         if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) {
02996                 /* FIXME: this implementation allows only a single QP
02997                  * per multicast group, and doesn't handle hash
02998                  * collisions.  Sufficient for IPoIB but may need to
02999                  * be extended in future.
03000                  */
03001                 DBGC ( hermon, "Hermon %p MGID index %#x already in use\n",
03002                        hermon, index );
03003                 return -EBUSY;
03004         }
03005 
03006         /* Update hash table entry */
03007         MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 );
03008         MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn );
03009         memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) );
03010         if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
03011                 DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
03012                        hermon, index, strerror ( rc ) );
03013                 return rc;
03014         }
03015 
03016         return 0;
03017 }
03018 
03019 /**
03020  * Detach from multicast group
03021  *
03022  * @v ibdev             Infiniband device
03023  * @v qp                Queue pair
03024  * @v gid               Multicast GID
03025  */
03026 static void hermon_mcast_detach ( struct ib_device *ibdev,
03027                                   struct ib_queue_pair *qp __unused,
03028                                   union ib_gid *gid ) {
03029         struct hermon *hermon = ib_get_drvdata ( ibdev );
03030         struct hermonprm_mgm_hash hash;
03031         struct hermonprm_mcg_entry mcg;
03032         unsigned int index;
03033         int rc;
03034 
03035         /* Generate hash table index */
03036         if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
03037                 DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
03038                        hermon, strerror ( rc ) );
03039                 return;
03040         }
03041         index = MLX_GET ( &hash, hash );
03042 
03043         /* Clear hash table entry */
03044         memset ( &mcg, 0, sizeof ( mcg ) );
03045         if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
03046                 DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
03047                        hermon, index, strerror ( rc ) );
03048                 return;
03049         }
03050 }
03051 
03052 /** Hermon Infiniband operations */
03053 static struct ib_device_operations hermon_ib_operations = {
03054         .create_cq      = hermon_create_cq,
03055         .destroy_cq     = hermon_destroy_cq,
03056         .create_qp      = hermon_create_qp,
03057         .modify_qp      = hermon_modify_qp,
03058         .destroy_qp     = hermon_destroy_qp,
03059         .post_send      = hermon_post_send,
03060         .post_recv      = hermon_post_recv,
03061         .poll_cq        = hermon_poll_cq,
03062         .poll_eq        = hermon_poll_eq,
03063         .open           = hermon_ib_open,
03064         .close          = hermon_ib_close,
03065         .mcast_attach   = hermon_mcast_attach,
03066         .mcast_detach   = hermon_mcast_detach,
03067         .set_port_info  = hermon_inform_sma,
03068         .set_pkey_table = hermon_inform_sma,
03069 };
03070 
03071 /**
03072  * Register Hermon Infiniband device
03073  *
03074  * @v hermon            Hermon device
03075  * @v port              Hermon port
03076  * @ret rc              Return status code
03077  */
03078 static int hermon_register_ibdev ( struct hermon *hermon,
03079                                    struct hermon_port *port ) {
03080         struct ib_device *ibdev = port->ibdev;
03081         int rc;
03082 
03083         /* Initialise parameters using SMC */
03084         ib_smc_init ( ibdev, hermon_mad );
03085 
03086         /* Register Infiniband device */
03087         if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
03088                 DBGC ( hermon, "Hermon %p port %d could not register IB "
03089                        "device: %s\n", hermon, ibdev->port, strerror ( rc ) );
03090                 return rc;
03091         }
03092 
03093         return 0;
03094 }
03095 
03096 /**
03097  * Handle Hermon Infiniband device port state change
03098  *
03099  * @v hermon            Hermon device
03100  * @v port              Hermon port
03101  * @v link_up           Link is up
03102  */
03103 static void hermon_state_change_ibdev ( struct hermon *hermon __unused,
03104                                         struct hermon_port *port,
03105                                         int link_up __unused ) {
03106         struct ib_device *ibdev = port->ibdev;
03107 
03108         /* Update MAD parameters */
03109         ib_smc_update ( ibdev, hermon_mad );
03110 }
03111 
03112 /**
03113  * Unregister Hermon Infiniband device
03114  *
03115  * @v hermon            Hermon device
03116  * @v port              Hermon port
03117  */
03118 static void hermon_unregister_ibdev ( struct hermon *hermon __unused,
03119                                       struct hermon_port *port ) {
03120         struct ib_device *ibdev = port->ibdev;
03121 
03122         unregister_ibdev ( ibdev );
03123 }
03124 
03125 /** Hermon Infiniband port type */
03126 static struct hermon_port_type hermon_port_type_ib = {
03127         .register_dev = hermon_register_ibdev,
03128         .state_change = hermon_state_change_ibdev,
03129         .unregister_dev = hermon_unregister_ibdev,
03130 };
03131 
03132 /***************************************************************************
03133  *
03134  * Ethernet operation
03135  *
03136  ***************************************************************************
03137  */
03138 
03139 /** Number of Hermon Ethernet send work queue entries */
03140 #define HERMON_ETH_NUM_SEND_WQES 2
03141 
03142 /** Number of Hermon Ethernet receive work queue entries */
03143 #define HERMON_ETH_NUM_RECV_WQES 4
03144 
03145 /** Number of Hermon Ethernet completion entries */
03146 #define HERMON_ETH_NUM_CQES 8
03147 
03148 /**
03149  * Transmit packet via Hermon Ethernet device
03150  *
03151  * @v netdev            Network device
03152  * @v iobuf             I/O buffer
03153  * @ret rc              Return status code
03154  */
03155 static int hermon_eth_transmit ( struct net_device *netdev,
03156                                  struct io_buffer *iobuf ) {
03157         struct hermon_port *port = netdev->priv;
03158         struct ib_device *ibdev = port->ibdev;
03159         struct hermon *hermon = ib_get_drvdata ( ibdev );
03160         int rc;
03161 
03162         /* Transmit packet */
03163         if ( ( rc = ib_post_send ( ibdev, port->eth_qp, NULL,
03164                                    iobuf ) ) != 0 ) {
03165                 DBGC ( hermon, "Hermon %p port %d could not transmit: %s\n",
03166                        hermon, ibdev->port, strerror ( rc ) );
03167                 return rc;
03168         }
03169 
03170         return 0;
03171 }
03172 
03173 /** Hermon Ethernet queue pair operations */
03174 static struct ib_queue_pair_operations hermon_eth_qp_op = {
03175         .alloc_iob = alloc_iob,
03176 };
03177 
03178 /**
03179  * Handle Hermon Ethernet device send completion
03180  *
03181  * @v ibdev             Infiniband device
03182  * @v qp                Queue pair
03183  * @v iobuf             I/O buffer
03184  * @v rc                Completion status code
03185  */
03186 static void hermon_eth_complete_send ( struct ib_device *ibdev __unused,
03187                                        struct ib_queue_pair *qp,
03188                                        struct io_buffer *iobuf, int rc ) {
03189         struct net_device *netdev = ib_qp_get_ownerdata ( qp );
03190 
03191         netdev_tx_complete_err ( netdev, iobuf, rc );
03192 }
03193 
03194 /**
03195  * Handle Hermon Ethernet device receive completion
03196  *
03197  * @v ibdev             Infiniband device
03198  * @v qp                Queue pair
03199  * @v dest              Destination address vector, or NULL
03200  * @v source            Source address vector, or NULL
03201  * @v iobuf             I/O buffer
03202  * @v rc                Completion status code
03203  */
03204 static void hermon_eth_complete_recv ( struct ib_device *ibdev __unused,
03205                                        struct ib_queue_pair *qp,
03206                                        struct ib_address_vector *dest __unused,
03207                                        struct ib_address_vector *source,
03208                                        struct io_buffer *iobuf, int rc ) {
03209         struct net_device *netdev = ib_qp_get_ownerdata ( qp );
03210         struct net_device *vlan;
03211 
03212         /* Find VLAN device, if applicable */
03213         if ( source->vlan_present ) {
03214                 if ( ( vlan = vlan_find ( netdev, source->vlan ) ) != NULL ) {
03215                         netdev = vlan;
03216                 } else if ( rc == 0 ) {
03217                         rc = -ENODEV;
03218                 }
03219         }
03220 
03221         /* Hand off to network layer */
03222         if ( rc == 0 ) {
03223                 netdev_rx ( netdev, iobuf );
03224         } else {
03225                 netdev_rx_err ( netdev, iobuf, rc );
03226         }
03227 }
03228 
03229 /** Hermon Ethernet device completion operations */
03230 static struct ib_completion_queue_operations hermon_eth_cq_op = {
03231         .complete_send = hermon_eth_complete_send,
03232         .complete_recv = hermon_eth_complete_recv,
03233 };
03234 
03235 /**
03236  * Poll Hermon Ethernet device
03237  *
03238  * @v netdev            Network device
03239  */
03240 static void hermon_eth_poll ( struct net_device *netdev ) {
03241         struct hermon_port *port = netdev->priv;
03242         struct ib_device *ibdev = port->ibdev;
03243 
03244         ib_poll_eq ( ibdev );
03245 }
03246 
03247 /**
03248  * Open Hermon Ethernet device
03249  *
03250  * @v netdev            Network device
03251  * @ret rc              Return status code
03252  */
03253 static int hermon_eth_open ( struct net_device *netdev ) {
03254         struct hermon_port *port = netdev->priv;
03255         struct ib_device *ibdev = port->ibdev;
03256         struct hermon *hermon = ib_get_drvdata ( ibdev );
03257         union hermonprm_set_port set_port;
03258         int rc;
03259 
03260         /* Open hardware */
03261         if ( ( rc = hermon_open ( hermon ) ) != 0 )
03262                 goto err_open;
03263 
03264         /* Allocate completion queue */
03265         if ( ( rc = ib_create_cq ( ibdev, HERMON_ETH_NUM_CQES,
03266                                    &hermon_eth_cq_op, &port->eth_cq ) ) != 0 ) {
03267                 DBGC ( hermon, "Hermon %p port %d could not create completion "
03268                        "queue: %s\n", hermon, ibdev->port, strerror ( rc ) );
03269                 goto err_create_cq;
03270         }
03271 
03272         /* Allocate queue pair */
03273         if ( ( rc = ib_create_qp ( ibdev, IB_QPT_ETH, HERMON_ETH_NUM_SEND_WQES,
03274                                    port->eth_cq, HERMON_ETH_NUM_RECV_WQES,
03275                                    port->eth_cq, &hermon_eth_qp_op,
03276                                    netdev->name, &port->eth_qp ) ) != 0 ) {
03277                 DBGC ( hermon, "Hermon %p port %d could not create queue "
03278                        "pair: %s\n", hermon, ibdev->port, strerror ( rc ) );
03279                 goto err_create_qp;
03280         }
03281         ib_qp_set_ownerdata ( port->eth_qp, netdev );
03282 
03283         /* Activate queue pair */
03284         if ( ( rc = ib_modify_qp ( ibdev, port->eth_qp ) ) != 0 ) {
03285                 DBGC ( hermon, "Hermon %p port %d could not modify queue "
03286                        "pair: %s\n", hermon, ibdev->port, strerror ( rc ) );
03287                 goto err_modify_qp;
03288         }
03289 
03290         /* Fill receive rings */
03291         ib_refill_recv ( ibdev, port->eth_qp );
03292 
03293         /* Set port general parameters */
03294         memset ( &set_port, 0, sizeof ( set_port ) );
03295         MLX_FILL_3 ( &set_port.general, 0,
03296                      v_mtu, 1,
03297                      v_pprx, 1,
03298                      v_pptx, 1 );
03299         MLX_FILL_1 ( &set_port.general, 1,
03300                      mtu, ( ETH_FRAME_LEN + 40 /* Used by card */ ) );
03301         MLX_FILL_1 ( &set_port.general, 2,
03302                      pfctx, ( 1 << FCOE_VLAN_PRIORITY ) );
03303         MLX_FILL_1 ( &set_port.general, 3,
03304                      pfcrx, ( 1 << FCOE_VLAN_PRIORITY ) );
03305         if ( ( rc = hermon_cmd_set_port ( hermon, 1,
03306                                           ( HERMON_SET_PORT_GENERAL_PARAM |
03307                                             ibdev->port ),
03308                                           &set_port ) ) != 0 ) {
03309                 DBGC ( hermon, "Hermon %p port %d could not set port general "
03310                        "parameters: %s\n",
03311                        hermon, ibdev->port, strerror ( rc ) );
03312                 goto err_set_port_general_params;
03313         }
03314 
03315         /* Set port receive QP */
03316         memset ( &set_port, 0, sizeof ( set_port ) );
03317         MLX_FILL_1 ( &set_port.rqp_calc, 0, base_qpn, port->eth_qp->qpn );
03318         MLX_FILL_1 ( &set_port.rqp_calc, 2,
03319                      mac_miss_index, 128 /* MAC misses go to promisc QP */ );
03320         MLX_FILL_2 ( &set_port.rqp_calc, 3,
03321                      vlan_miss_index, 127 /* VLAN misses go to promisc QP */,
03322                      no_vlan_index, 126 /* VLAN-free go to promisc QP */ );
03323         MLX_FILL_2 ( &set_port.rqp_calc, 5,
03324                      promisc_qpn, port->eth_qp->qpn,
03325                      en_uc_promisc, 1 );
03326         MLX_FILL_2 ( &set_port.rqp_calc, 6,
03327                      def_mcast_qpn, port->eth_qp->qpn,
03328                      mc_promisc_mode, 2 /* Receive all multicasts */ );
03329         if ( ( rc = hermon_cmd_set_port ( hermon, 1,
03330                                           ( HERMON_SET_PORT_RECEIVE_QP |
03331                                             ibdev->port ),
03332                                           &set_port ) ) != 0 ) {
03333                 DBGC ( hermon, "Hermon %p port %d could not set port receive "
03334                        "QP: %s\n", hermon, ibdev->port, strerror ( rc ) );
03335                 goto err_set_port_receive_qp;
03336         }
03337 
03338         /* Initialise port */
03339         if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
03340                 DBGC ( hermon, "Hermon %p port %d could not initialise port: "
03341                        "%s\n", hermon, ibdev->port, strerror ( rc ) );
03342                 goto err_init_port;
03343         }
03344 
03345         return 0;
03346 
03347  err_init_port:
03348  err_set_port_receive_qp:
03349  err_set_port_general_params:
03350  err_modify_qp:
03351         ib_destroy_qp ( ibdev, port->eth_qp );
03352  err_create_qp:
03353         ib_destroy_cq ( ibdev, port->eth_cq );
03354  err_create_cq:
03355         hermon_close ( hermon );
03356  err_open:
03357         return rc;
03358 }
03359 
03360 /**
03361  * Close Hermon Ethernet device
03362  *
03363  * @v netdev            Network device
03364  */
03365 static void hermon_eth_close ( struct net_device *netdev ) {
03366         struct hermon_port *port = netdev->priv;
03367         struct ib_device *ibdev = port->ibdev;
03368         struct hermon *hermon = ib_get_drvdata ( ibdev );
03369         int rc;
03370 
03371         /* Close port */
03372         if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
03373                 DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
03374                        hermon, ibdev->port, strerror ( rc ) );
03375                 /* Nothing we can do about this */
03376         }
03377 
03378         /* Tear down the queues */
03379         ib_destroy_qp ( ibdev, port->eth_qp );
03380         ib_destroy_cq ( ibdev, port->eth_cq );
03381 
03382         /* Close hardware */
03383         hermon_close ( hermon );
03384 }
03385 
03386 /** Hermon Ethernet network device operations */
03387 static struct net_device_operations hermon_eth_operations = {
03388         .open           = hermon_eth_open,
03389         .close          = hermon_eth_close,
03390         .transmit       = hermon_eth_transmit,
03391         .poll           = hermon_eth_poll,
03392 };
03393 
03394 /**
03395  * Register Hermon Ethernet device
03396  *
03397  * @v hermon            Hermon device
03398  * @v port              Hermon port
03399  * @ret rc              Return status code
03400  */
03401 static int hermon_register_netdev ( struct hermon *hermon,
03402                                     struct hermon_port *port ) {
03403         struct net_device *netdev = port->netdev;
03404         struct ib_device *ibdev = port->ibdev;
03405         struct hermonprm_query_port_cap query_port;
03406         union {
03407                 uint8_t bytes[8];
03408                 uint32_t dwords[2];
03409         } mac;
03410         int rc;
03411 
03412         /* Retrieve MAC address */
03413         if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port,
03414                                             &query_port ) ) != 0 ) {
03415                 DBGC ( hermon, "Hermon %p port %d could not query port: %s\n",
03416                        hermon, ibdev->port, strerror ( rc ) );
03417                 goto err_query_port;
03418         }
03419         mac.dwords[0] = htonl ( MLX_GET ( &query_port, mac_47_32 ) );
03420         mac.dwords[1] = htonl ( MLX_GET ( &query_port, mac_31_0 ) );
03421         memcpy ( netdev->hw_addr,
03422                  &mac.bytes[ sizeof ( mac.bytes ) - ETH_ALEN ], ETH_ALEN );
03423 
03424         /* Register network device */
03425         if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
03426                 DBGC ( hermon, "Hermon %p port %d could not register network "
03427                        "device: %s\n", hermon, ibdev->port, strerror ( rc ) );
03428                 goto err_register_netdev;
03429         }
03430 
03431         /* Register non-volatile options */
03432         if ( ( rc = register_nvo ( &port->nvo,
03433                                    netdev_settings ( netdev ) ) ) != 0 ) {
03434                 DBGC ( hermon, "Hermon %p port %d could not register non-"
03435                        "volatile options: %s\n",
03436                        hermon, ibdev->port, strerror ( rc ) );
03437                 goto err_register_nvo;
03438         }
03439 
03440         return 0;
03441 
03442         unregister_nvo ( &port->nvo );
03443  err_register_nvo:
03444         unregister_netdev ( netdev );
03445  err_register_netdev:
03446  err_query_port:
03447         return rc;
03448 }
03449 
03450 /**
03451  * Handle Hermon Ethernet device port state change
03452  *
03453  * @v hermon            Hermon device
03454  * @v port              Hermon port
03455  * @v link_up           Link is up
03456  */
03457 static void hermon_state_change_netdev ( struct hermon *hermon __unused,
03458                                          struct hermon_port *port,
03459                                          int link_up ) {
03460         struct net_device *netdev = port->netdev;
03461 
03462         if ( link_up ) {
03463                 netdev_link_up ( netdev );
03464         } else {
03465                 netdev_link_down ( netdev );
03466         }
03467 }
03468 
03469 /**
03470  * Unregister Hermon Ethernet device
03471  *
03472  * @v hermon            Hermon device
03473  * @v port              Hermon port
03474  */
03475 static void hermon_unregister_netdev ( struct hermon *hermon __unused,
03476                                        struct hermon_port *port ) {
03477         struct net_device *netdev = port->netdev;
03478 
03479         unregister_nvo ( &port->nvo );
03480         unregister_netdev ( netdev );
03481 }
03482 
03483 /** Hermon Ethernet port type */
03484 static struct hermon_port_type hermon_port_type_eth = {
03485         .register_dev = hermon_register_netdev,
03486         .state_change = hermon_state_change_netdev,
03487         .unregister_dev = hermon_unregister_netdev,
03488 };
03489 
03490 /***************************************************************************
03491  *
03492  * Port type detection
03493  *
03494  ***************************************************************************
03495  */
03496 
03497 /** Timeout for port sensing */
03498 #define HERMON_SENSE_PORT_TIMEOUT ( TICKS_PER_SEC / 2 )
03499 
03500 /**
03501  * Name port type
03502  *
03503  * @v port_type         Port type
03504  * @v port_type_name    Port type name
03505  */
03506 static inline const char * hermon_name_port_type ( unsigned int port_type ) {
03507         switch ( port_type ) {
03508         case HERMON_PORT_TYPE_UNKNOWN:  return "unknown";
03509         case HERMON_PORT_TYPE_IB:       return "Infiniband";
03510         case HERMON_PORT_TYPE_ETH:      return "Ethernet";
03511         default:                        return "INVALID";
03512         }
03513 }
03514 
03515 /**
03516  * Sense port type
03517  *
03518  * @v hermon            Hermon device
03519  * @v port              Hermon port
03520  * @ret port_type       Port type, or negative error
03521  */
03522 static int hermon_sense_port_type ( struct hermon *hermon,
03523                                     struct hermon_port *port ) {
03524         struct ib_device *ibdev = port->ibdev;
03525         struct hermonprm_sense_port sense_port;
03526         int port_type;
03527         int rc;
03528 
03529         /* If DPDP is not supported, always assume Infiniband */
03530         if ( ! hermon->cap.dpdp ) {
03531                 port_type = HERMON_PORT_TYPE_IB;
03532                 DBGC ( hermon, "Hermon %p port %d does not support DPDP; "
03533                        "assuming an %s network\n", hermon, ibdev->port,
03534                        hermon_name_port_type ( port_type ) );
03535                 return port_type;
03536         }
03537 
03538         /* Sense the port type */
03539         if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
03540                                             &sense_port ) ) != 0 ) {
03541                 DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
03542                        hermon, ibdev->port, strerror ( rc ) );
03543                 return rc;
03544         }
03545         port_type = MLX_GET ( &sense_port, port_type );
03546 
03547         DBGC ( hermon, "Hermon %p port %d sensed an %s network\n",
03548                hermon, ibdev->port, hermon_name_port_type ( port_type ) );
03549         return port_type;
03550 }
03551 
03552 /**
03553  * Set port type
03554  *
03555  * @v hermon            Hermon device
03556  * @v port              Hermon port
03557  * @ret rc              Return status code
03558  */
03559 static int hermon_set_port_type ( struct hermon *hermon,
03560                                   struct hermon_port *port ) {
03561         struct ib_device *ibdev = port->ibdev;
03562         struct hermonprm_query_port_cap query_port;
03563         int ib_supported;
03564         int eth_supported;
03565         int port_type;
03566         unsigned long start;
03567         unsigned long elapsed;
03568         int rc;
03569 
03570         /* Check to see which types are supported */
03571         if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port,
03572                                             &query_port ) ) != 0 ) {
03573                 DBGC ( hermon, "Hermon %p port %d could not query port: %s\n",
03574                        hermon, ibdev->port, strerror ( rc ) );
03575                 return rc;
03576         }
03577         ib_supported = MLX_GET ( &query_port, ib );
03578         eth_supported = MLX_GET ( &query_port, eth );
03579         DBGC ( hermon, "Hermon %p port %d supports%s%s%s\n",
03580                hermon, ibdev->port, ( ib_supported ? " Infiniband" : "" ),
03581                ( ( ib_supported && eth_supported ) ? " and" : "" ),
03582                ( eth_supported ? " Ethernet" : "" ) );
03583 
03584         /* Sense network, if applicable */
03585         if ( ib_supported && eth_supported ) {
03586 
03587                 /* Both types are supported; try sensing network */
03588                 start = currticks();
03589                 do {
03590                         /* Try sensing port */
03591                         port_type = hermon_sense_port_type ( hermon, port );
03592                         if ( port_type < 0 ) {
03593                                 rc = port_type;
03594                                 return rc;
03595                         }
03596                 } while ( ( port_type == HERMON_PORT_TYPE_UNKNOWN ) &&
03597                           ( ( elapsed = ( currticks() - start ) ) <
03598                             HERMON_SENSE_PORT_TIMEOUT ) );
03599 
03600                 /* Set port type based on sensed network, defaulting
03601                  * to Infiniband if nothing was sensed.
03602                  */
03603                 switch ( port_type ) {
03604                 case HERMON_PORT_TYPE_ETH:
03605                         port->type = &hermon_port_type_eth;
03606                         break;
03607                 case HERMON_PORT_TYPE_IB:
03608                 case HERMON_PORT_TYPE_UNKNOWN:
03609                         port->type = &hermon_port_type_ib;
03610                         break;
03611                 default:
03612                         return -EINVAL;
03613                 }
03614 
03615         } else if ( eth_supported ) {
03616                 port->type = &hermon_port_type_eth;
03617         } else {
03618                 port->type = &hermon_port_type_ib;
03619         }
03620 
03621         assert ( port->type != NULL );
03622         return 0;
03623 }
03624 
03625 /***************************************************************************
03626  *
03627  * BOFM interface
03628  *
03629  ***************************************************************************
03630  */
03631 
03632 /**
03633  * Harvest Ethernet MAC for BOFM
03634  *
03635  * @v bofm              BOFM device
03636  * @v mport             Multi-port index
03637  * @v mac               MAC to fill in
03638  * @ret rc              Return status code
03639  */
03640 static int hermon_bofm_harvest ( struct bofm_device *bofm, unsigned int mport,
03641                                  uint8_t *mac ) {
03642         struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
03643         struct hermonprm_mod_stat_cfg stat_cfg;
03644         union {
03645                 uint8_t bytes[8];
03646                 uint32_t dwords[2];
03647         } buf;
03648         int rc;
03649 
03650         /* Query static configuration */
03651         if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
03652                                           HERMON_MOD_STAT_CFG_QUERY,
03653                                           HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
03654                                           &stat_cfg ) ) != 0 ) {
03655                 DBGC ( hermon, "Hermon %p port %d could not query "
03656                        "configuration: %s\n", hermon, mport, strerror ( rc ) );
03657                 return rc;
03658         }
03659 
03660         /* Retrieve MAC address */
03661         buf.dwords[0] = htonl ( MLX_GET ( &stat_cfg, mac_high ) );
03662         buf.dwords[1] = htonl ( MLX_GET ( &stat_cfg, mac_low ) );
03663         memcpy ( mac, &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ],
03664                  ETH_ALEN );
03665 
03666         DBGC ( hermon, "Hermon %p port %d harvested MAC address %s\n",
03667                hermon, mport, eth_ntoa ( mac ) );
03668 
03669         return 0;
03670 }
03671 
03672 /**
03673  * Update Ethernet MAC for BOFM
03674  *
03675  * @v bofm              BOFM device
03676  * @v mport             Multi-port index
03677  * @v mac               MAC to fill in
03678  * @ret rc              Return status code
03679  */
03680 static int hermon_bofm_update ( struct bofm_device *bofm, unsigned int mport,
03681                                 const uint8_t *mac ) {
03682         struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
03683         struct hermonprm_mod_stat_cfg stat_cfg;
03684         union {
03685                 uint8_t bytes[8];
03686                 uint32_t dwords[2];
03687         } buf;
03688         int rc;
03689 
03690         /* Prepare MAC address */
03691         memset ( &buf, 0, sizeof ( buf ) );
03692         memcpy ( &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ], mac,
03693                  ETH_ALEN );
03694 
03695         /* Modify static configuration */
03696         memset ( &stat_cfg, 0, sizeof ( stat_cfg ) );
03697         MLX_FILL_2 ( &stat_cfg, 36,
03698                      mac_m, 1,
03699                      mac_high, ntohl ( buf.dwords[0] ) );
03700         MLX_FILL_1 ( &stat_cfg, 37, mac_low, ntohl ( buf.dwords[1] ) );
03701         if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
03702                                           HERMON_MOD_STAT_CFG_SET,
03703                                           HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
03704                                           &stat_cfg ) ) != 0 ) {
03705                 DBGC ( hermon, "Hermon %p port %d could not modify "
03706                        "configuration: %s\n", hermon, mport, strerror ( rc ) );
03707                 return rc;
03708         }
03709 
03710         DBGC ( hermon, "Hermon %p port %d updated MAC address to %s\n",
03711                hermon, mport, eth_ntoa ( mac ) );
03712 
03713         return 0;
03714 }
03715 
03716 /** Hermon BOFM operations */
03717 static struct bofm_operations hermon_bofm_operations = {
03718         .harvest = hermon_bofm_harvest,
03719         .update = hermon_bofm_update,
03720 };
03721 
03722 /***************************************************************************
03723  *
03724  * PCI interface
03725  *
03726  ***************************************************************************
03727  */
03728 
03729 /**
03730  * Allocate Hermon device
03731  *
03732  * @v pci               PCI device
03733  * @v id                PCI ID
03734  * @ret rc              Return status code
03735  */
03736 static struct hermon * hermon_alloc ( void ) {
03737         struct hermon *hermon;
03738 
03739         /* Allocate Hermon device */
03740         hermon = zalloc ( sizeof ( *hermon ) );
03741         if ( ! hermon )
03742                 goto err_hermon;
03743 
03744         /* Allocate space for mailboxes */
03745         hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE,
03746                                           HERMON_MBOX_ALIGN );
03747         if ( ! hermon->mailbox_in )
03748                 goto err_mailbox_in;
03749         hermon->mailbox_out = malloc_dma ( HERMON_MBOX_SIZE,
03750                                            HERMON_MBOX_ALIGN );
03751         if ( ! hermon->mailbox_out )
03752                 goto err_mailbox_out;
03753 
03754         return hermon;
03755 
03756         free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
03757  err_mailbox_out:
03758         free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
03759  err_mailbox_in:
03760         free ( hermon );
03761  err_hermon:
03762         return NULL;
03763 }
03764 
03765 /**
03766  * Free Hermon device
03767  *
03768  * @v hermon            Hermon device
03769  */
03770 static void hermon_free ( struct hermon *hermon ) {
03771 
03772         ufree ( hermon->icm );
03773         ufree ( hermon->firmware_area );
03774         free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
03775         free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
03776         free ( hermon );
03777 }
03778 
03779 /**
03780  * Probe PCI device
03781  *
03782  * @v pci               PCI device
03783  * @v id                PCI ID
03784  * @ret rc              Return status code
03785  */
03786 static int hermon_probe ( struct pci_device *pci ) {
03787         struct hermon *hermon;
03788         struct ib_device *ibdev;
03789         struct net_device *netdev;
03790         struct hermon_port *port;
03791         unsigned int i;
03792         int rc;
03793 
03794         /* Allocate Hermon device */
03795         hermon = hermon_alloc();
03796         if ( ! hermon ) {
03797                 rc = -ENOMEM;
03798                 goto err_alloc;
03799         }
03800         pci_set_drvdata ( pci, hermon );
03801         hermon->pci = pci;
03802 
03803         /* Fix up PCI device */
03804         adjust_pci_device ( pci );
03805 
03806         /* Map PCI BARs */
03807         hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR ),
03808                                    HERMON_PCI_CONFIG_BAR_SIZE );
03809         hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ),
03810                                 HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE );
03811 
03812         /* Reset device */
03813         hermon_reset ( hermon );
03814 
03815         /* Start firmware */
03816         if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 )
03817                 goto err_start_firmware;
03818 
03819         /* Get device limits */
03820         if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
03821                 goto err_get_cap;
03822 
03823         /* Allocate Infiniband devices */
03824         for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
03825                 ibdev = alloc_ibdev ( 0 );
03826                 if ( ! ibdev ) {
03827                         rc = -ENOMEM;
03828                         goto err_alloc_ibdev;
03829                 }
03830                 hermon->port[i].ibdev = ibdev;
03831                 ibdev->op = &hermon_ib_operations;
03832                 ibdev->dev = &pci->dev;
03833                 ibdev->port = ( HERMON_PORT_BASE + i );
03834                 ib_set_drvdata ( ibdev, hermon );
03835         }
03836 
03837         /* Allocate network devices */
03838         for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
03839                 netdev = alloc_etherdev ( 0 );
03840                 if ( ! netdev ) {
03841                         rc = -ENOMEM;
03842                         goto err_alloc_netdev;
03843                 }
03844                 hermon->port[i].netdev = netdev;
03845                 netdev_init ( netdev, &hermon_eth_operations );
03846                 netdev->dev = &pci->dev;
03847                 netdev->priv = &hermon->port[i];
03848         }
03849 
03850         /* Start device */
03851         if ( ( rc = hermon_start ( hermon, 1 ) ) != 0 )
03852                 goto err_start;
03853 
03854         /* Determine port types */
03855         for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
03856                 port = &hermon->port[i];
03857                 if ( ( rc = hermon_set_port_type ( hermon, port ) ) != 0 )
03858                         goto err_set_port_type;
03859         }
03860 
03861         /* Initialise non-volatile storage */
03862         nvs_vpd_init ( &hermon->nvsvpd, pci );
03863         for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
03864                 port = &hermon->port[i];
03865                 nvs_vpd_nvo_init ( &hermon->nvsvpd,
03866                                    HERMON_VPD_FIELD ( port->ibdev->port ),
03867                                    &port->nvo, NULL );
03868         }
03869 
03870         /* Register devices */
03871         for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
03872                 port = &hermon->port[i];
03873                 if ( ( rc = port->type->register_dev ( hermon, port ) ) != 0 )
03874                         goto err_register;
03875         }
03876 
03877         /* Leave device quiescent until opened */
03878         if ( hermon->open_count == 0 )
03879                 hermon_stop ( hermon );
03880 
03881         return 0;
03882 
03883         i = hermon->cap.num_ports;
03884  err_register:
03885         for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
03886                 port = &hermon->port[i];
03887                 port->type->unregister_dev ( hermon, port );
03888         }
03889  err_set_port_type:
03890         hermon_stop ( hermon );
03891  err_start:
03892         i = hermon->cap.num_ports;
03893  err_alloc_netdev:
03894         for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
03895                 netdev_nullify ( hermon->port[i].netdev );
03896                 netdev_put ( hermon->port[i].netdev );
03897         }
03898         i = hermon->cap.num_ports;
03899  err_alloc_ibdev:
03900         for ( i-- ; ( signed int ) i >= 0 ; i-- )
03901                 ibdev_put ( hermon->port[i].ibdev );
03902  err_get_cap:
03903         hermon_stop_firmware ( hermon );
03904  err_start_firmware:
03905         iounmap ( hermon->uar );
03906         iounmap ( hermon->config );
03907         hermon_free ( hermon );
03908  err_alloc:
03909         return rc;
03910 }
03911 
03912 /**
03913  * Remove PCI device
03914  *
03915  * @v pci               PCI device
03916  */
03917 static void hermon_remove ( struct pci_device *pci ) {
03918         struct hermon *hermon = pci_get_drvdata ( pci );
03919         struct hermon_port *port;
03920         int i;
03921 
03922         for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
03923                 port = &hermon->port[i];
03924                 port->type->unregister_dev ( hermon, port );
03925         }
03926         for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
03927                 netdev_nullify ( hermon->port[i].netdev );
03928                 netdev_put ( hermon->port[i].netdev );
03929         }
03930         for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
03931                 ibdev_put ( hermon->port[i].ibdev );
03932         iounmap ( hermon->uar );
03933         iounmap ( hermon->config );
03934         hermon_free ( hermon );
03935 }
03936 
03937 /**
03938  * Probe PCI device for BOFM
03939  *
03940  * @v pci               PCI device
03941  * @v id                PCI ID
03942  * @ret rc              Return status code
03943  */
03944 static int hermon_bofm_probe ( struct pci_device *pci ) {
03945         struct hermon *hermon;
03946         int rc;
03947 
03948         /* Allocate Hermon device */
03949         hermon = hermon_alloc();
03950         if ( ! hermon ) {
03951                 rc = -ENOMEM;
03952                 goto err_alloc;
03953         }
03954         pci_set_drvdata ( pci, hermon );
03955         hermon->pci = pci;
03956 
03957         /* Fix up PCI device */
03958         adjust_pci_device ( pci );
03959 
03960         /* Map PCI BAR */
03961         hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR ),
03962                                    HERMON_PCI_CONFIG_BAR_SIZE );
03963 
03964         /* Initialise BOFM device */
03965         bofm_init ( &hermon->bofm, pci, &hermon_bofm_operations );
03966 
03967         /* Register BOFM device */
03968         if ( ( rc = bofm_register ( &hermon->bofm ) ) != 0 ) {
03969                 DBGC ( hermon, "Hermon %p could not register BOFM device: "
03970                        "%s\n", hermon, strerror ( rc ) );
03971                 goto err_bofm_register;
03972         }
03973 
03974         return 0;
03975 
03976  err_bofm_register:
03977         iounmap ( hermon->config );
03978         hermon_free ( hermon );
03979  err_alloc:
03980         return rc;
03981 }
03982 
03983 /**
03984  * Remove PCI device for BOFM
03985  *
03986  * @v pci               PCI device
03987  */
03988 static void hermon_bofm_remove ( struct pci_device *pci ) {
03989         struct hermon *hermon = pci_get_drvdata ( pci );
03990 
03991         bofm_unregister ( &hermon->bofm );
03992         iounmap ( hermon->config );
03993         hermon_free ( hermon );
03994 }
03995 
03996 static struct pci_device_id hermon_nics[] = {
03997         PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ),
03998         PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ),
03999         PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ),
04000         PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ),
04001         PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ),
04002         PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
04003         PCI_ROM ( 0x15b3, 0x6368, "mt25448", "MT25448 HCA driver", 0 ),
04004         PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ),
04005         PCI_ROM ( 0x15b3, 0x6372, "mt25458", "MT25458 HCA driver", 0 ),
04006         PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ),
04007         PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ),
04008         PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ),
04009 };
04010 
04011 struct pci_driver hermon_driver __pci_driver = {
04012         .ids = hermon_nics,
04013         .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
04014         .probe = hermon_probe,
04015         .remove = hermon_remove,
04016 };
04017 
04018 struct pci_driver hermon_bofm_driver __bofm_driver = {
04019         .ids = hermon_nics,
04020         .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
04021         .probe = hermon_bofm_probe,
04022         .remove = hermon_bofm_remove,
04023 };