iPXE
arbel.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * Based in part upon the original driver by Mellanox Technologies
00005  * Ltd.  Portions may be Copyright (c) Mellanox Technologies Ltd.
00006  *
00007  * This program is free software; you can redistribute it and/or
00008  * modify it under the terms of the GNU General Public License as
00009  * published by the Free Software Foundation; either version 2 of the
00010  * License, or any later version.
00011  *
00012  * This program is distributed in the hope that it will be useful, but
00013  * WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015  * General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU General Public License
00018  * along with this program; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00020  * 02110-1301, USA.
00021  *
00022  * You can also choose to distribute this program under the terms of
00023  * the Unmodified Binary Distribution Licence (as given in the file
00024  * COPYING.UBDL), provided that you have satisfied its requirements.
00025  */
00026 
00027 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00028 
00029 #include <stdint.h>
00030 #include <stdlib.h>
00031 #include <stdio.h>
00032 #include <string.h>
00033 #include <strings.h>
00034 #include <unistd.h>
00035 #include <errno.h>
00036 #include <byteswap.h>
00037 #include <ipxe/io.h>
00038 #include <ipxe/pci.h>
00039 #include <ipxe/pcibackup.h>
00040 #include <ipxe/malloc.h>
00041 #include <ipxe/umalloc.h>
00042 #include <ipxe/iobuf.h>
00043 #include <ipxe/netdevice.h>
00044 #include <ipxe/infiniband.h>
00045 #include <ipxe/ib_smc.h>
00046 #include "arbel.h"
00047 
00048 /**
00049  * @file
00050  *
00051  * Mellanox Arbel Infiniband HCA
00052  *
00053  */
00054 
00055 /***************************************************************************
00056  *
00057  * Queue number allocation
00058  *
00059  ***************************************************************************
00060  */
00061 
00062 /**
00063  * Allocate offset within usage bitmask
00064  *
00065  * @v bits              Usage bitmask
00066  * @v bits_len          Length of usage bitmask
00067  * @ret bit             First free bit within bitmask, or negative error
00068  */
00069 static int arbel_bitmask_alloc ( arbel_bitmask_t *bits,
00070                                  unsigned int bits_len ) {
00071         unsigned int bit = 0;
00072         arbel_bitmask_t mask = 1;
00073 
00074         while ( bit < bits_len ) {
00075                 if ( ( mask & *bits ) == 0 ) {
00076                         *bits |= mask;
00077                         return bit;
00078                 }
00079                 bit++;
00080                 mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) );
00081                 if ( mask == 1 )
00082                         bits++;
00083         }
00084         return -ENFILE;
00085 }
00086 
00087 /**
00088  * Free offset within usage bitmask
00089  *
00090  * @v bits              Usage bitmask
00091  * @v bit               Bit within bitmask
00092  */
00093 static void arbel_bitmask_free ( arbel_bitmask_t *bits, int bit ) {
00094         arbel_bitmask_t mask;
00095 
00096         mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) );
00097         bits += ( bit / ( 8 * sizeof ( mask ) ) );
00098         *bits &= ~mask;
00099 }
00100 
00101 /***************************************************************************
00102  *
00103  * HCA commands
00104  *
00105  ***************************************************************************
00106  */
00107 
00108 /**
00109  * Wait for Arbel command completion
00110  *
00111  * @v arbel             Arbel device
00112  * @ret rc              Return status code
00113  */
00114 static int arbel_cmd_wait ( struct arbel *arbel,
00115                             struct arbelprm_hca_command_register *hcr ) {
00116         unsigned int wait;
00117 
00118         for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
00119                 hcr->u.dwords[6] =
00120                         readl ( arbel->config + ARBEL_HCR_REG ( 6 ) );
00121                 if ( MLX_GET ( hcr, go ) == 0 )
00122                         return 0;
00123                 mdelay ( 1 );
00124         }
00125         return -EBUSY;
00126 }
00127 
00128 /**
00129  * Issue HCA command
00130  *
00131  * @v arbel             Arbel device
00132  * @v command           Command opcode, flags and input/output lengths
00133  * @v op_mod            Opcode modifier (0 if no modifier applicable)
00134  * @v in                Input parameters
00135  * @v in_mod            Input modifier (0 if no modifier applicable)
00136  * @v out               Output parameters
00137  * @ret rc              Return status code
00138  */
00139 static int arbel_cmd ( struct arbel *arbel, unsigned long command,
00140                        unsigned int op_mod, const void *in,
00141                        unsigned int in_mod, void *out ) {
00142         struct arbelprm_hca_command_register hcr;
00143         unsigned int opcode = ARBEL_HCR_OPCODE ( command );
00144         size_t in_len = ARBEL_HCR_IN_LEN ( command );
00145         size_t out_len = ARBEL_HCR_OUT_LEN ( command );
00146         void *in_buffer;
00147         void *out_buffer;
00148         unsigned int status;
00149         unsigned int i;
00150         int rc;
00151 
00152         assert ( in_len <= ARBEL_MBOX_SIZE );
00153         assert ( out_len <= ARBEL_MBOX_SIZE );
00154 
00155         DBGC2 ( arbel, "Arbel %p command %02x in %zx%s out %zx%s\n",
00156                 arbel, opcode, in_len,
00157                 ( ( command & ARBEL_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
00158                 ( ( command & ARBEL_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
00159 
00160         /* Check that HCR is free */
00161         if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
00162                 DBGC ( arbel, "Arbel %p command interface locked\n", arbel );
00163                 return rc;
00164         }
00165 
00166         /* Prepare HCR */
00167         memset ( &hcr, 0, sizeof ( hcr ) );
00168         in_buffer = &hcr.u.dwords[0];
00169         if ( in_len && ( command & ARBEL_HCR_IN_MBOX ) ) {
00170                 in_buffer = arbel->mailbox_in;
00171                 MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) );
00172                 MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
00173         }
00174         memcpy ( in_buffer, in, in_len );
00175         MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
00176         out_buffer = &hcr.u.dwords[3];
00177         if ( out_len && ( command & ARBEL_HCR_OUT_MBOX ) ) {
00178                 out_buffer = arbel->mailbox_out;
00179                 MLX_FILL_H ( &hcr, 3, out_param_h,
00180                              virt_to_bus ( out_buffer ) );
00181                 MLX_FILL_1 ( &hcr, 4, out_param_l,
00182                              virt_to_bus ( out_buffer ) );
00183         }
00184         MLX_FILL_3 ( &hcr, 6,
00185                      opcode, opcode,
00186                      opcode_modifier, op_mod,
00187                      go, 1 );
00188         DBGC ( arbel, "Arbel %p issuing command %04x\n", arbel, opcode );
00189         DBGC2_HDA ( arbel, virt_to_phys ( arbel->config + ARBEL_HCR_BASE ),
00190                     &hcr, sizeof ( hcr ) );
00191         if ( in_len && ( command & ARBEL_HCR_IN_MBOX ) ) {
00192                 DBGC2 ( arbel, "Input mailbox:\n" );
00193                 DBGC2_HDA ( arbel, virt_to_phys ( in_buffer ), in_buffer,
00194                             ( ( in_len < 512 ) ? in_len : 512 ) );
00195         }
00196 
00197         /* Issue command */
00198         for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
00199               i++ ) {
00200                 writel ( hcr.u.dwords[i],
00201                          arbel->config + ARBEL_HCR_REG ( i ) );
00202                 barrier();
00203         }
00204 
00205         /* Wait for command completion */
00206         if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
00207                 DBGC ( arbel, "Arbel %p timed out waiting for command:\n",
00208                        arbel );
00209                 DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
00210                 return rc;
00211         }
00212 
00213         /* Check command status */
00214         status = MLX_GET ( &hcr, status );
00215         if ( status != 0 ) {
00216                 DBGC ( arbel, "Arbel %p command failed with status %02x:\n",
00217                        arbel, status );
00218                 DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
00219                 return -EIO;
00220         }
00221 
00222         /* Read output parameters, if any */
00223         hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) );
00224         hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) );
00225         memcpy ( out, out_buffer, out_len );
00226         if ( out_len ) {
00227                 DBGC2 ( arbel, "Output%s:\n",
00228                         ( command & ARBEL_HCR_OUT_MBOX ) ? " mailbox" : "" );
00229                 DBGC2_HDA ( arbel, virt_to_phys ( out_buffer ), out_buffer,
00230                             ( ( out_len < 512 ) ? out_len : 512 ) );
00231         }
00232 
00233         return 0;
00234 }
00235 
00236 static inline int
00237 arbel_cmd_query_dev_lim ( struct arbel *arbel,
00238                           struct arbelprm_query_dev_lim *dev_lim ) {
00239         return arbel_cmd ( arbel,
00240                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_DEV_LIM,
00241                                                1, sizeof ( *dev_lim ) ),
00242                            0, NULL, 0, dev_lim );
00243 }
00244 
00245 static inline int
00246 arbel_cmd_query_fw ( struct arbel *arbel, struct arbelprm_query_fw *fw ) {
00247         return arbel_cmd ( arbel,
00248                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_FW, 
00249                                                1, sizeof ( *fw ) ),
00250                            0, NULL, 0, fw );
00251 }
00252 
00253 static inline int
00254 arbel_cmd_init_hca ( struct arbel *arbel,
00255                      const struct arbelprm_init_hca *init_hca ) {
00256         return arbel_cmd ( arbel,
00257                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT_HCA,
00258                                               1, sizeof ( *init_hca ) ),
00259                            0, init_hca, 0, NULL );
00260 }
00261 
00262 static inline int
00263 arbel_cmd_close_hca ( struct arbel *arbel ) {
00264         return arbel_cmd ( arbel,
00265                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CLOSE_HCA ),
00266                            0, NULL, 0, NULL );
00267 }
00268 
00269 static inline int
00270 arbel_cmd_init_ib ( struct arbel *arbel, unsigned int port,
00271                     const struct arbelprm_init_ib *init_ib ) {
00272         return arbel_cmd ( arbel,
00273                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT_IB,
00274                                               1, sizeof ( *init_ib ) ),
00275                            0, init_ib, port, NULL );
00276 }
00277 
00278 static inline int
00279 arbel_cmd_close_ib ( struct arbel *arbel, unsigned int port ) {
00280         return arbel_cmd ( arbel,
00281                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CLOSE_IB ),
00282                            0, NULL, port, NULL );
00283 }
00284 
00285 static inline int
00286 arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index,
00287                       const struct arbelprm_mpt *mpt ) {
00288         return arbel_cmd ( arbel,
00289                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_MPT,
00290                                               1, sizeof ( *mpt ) ),
00291                            0, mpt, index, NULL );
00292 }
00293 
00294 static inline int
00295 arbel_cmd_map_eq ( struct arbel *arbel, unsigned long index_map,
00296                    const struct arbelprm_event_mask *mask ) {
00297         return arbel_cmd ( arbel,
00298                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_MAP_EQ,
00299                                               0, sizeof ( *mask ) ),
00300                            0, mask, index_map, NULL );
00301 }
00302 
00303 static inline int
00304 arbel_cmd_sw2hw_eq ( struct arbel *arbel, unsigned int index,
00305                      const struct arbelprm_eqc *eqctx ) {
00306         return arbel_cmd ( arbel,
00307                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_EQ,
00308                                               1, sizeof ( *eqctx ) ),
00309                            0, eqctx, index, NULL );
00310 }
00311 
00312 static inline int
00313 arbel_cmd_hw2sw_eq ( struct arbel *arbel, unsigned int index,
00314                      struct arbelprm_eqc *eqctx ) {
00315         return arbel_cmd ( arbel,
00316                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_HW2SW_EQ,
00317                                                1, sizeof ( *eqctx ) ),
00318                            1, NULL, index, eqctx );
00319 }
00320 
00321 static inline int
00322 arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
00323                      const struct arbelprm_completion_queue_context *cqctx ) {
00324         return arbel_cmd ( arbel,
00325                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_CQ,
00326                                               1, sizeof ( *cqctx ) ),
00327                            0, cqctx, cqn, NULL );
00328 }
00329 
00330 static inline int
00331 arbel_cmd_hw2sw_cq ( struct arbel *arbel, unsigned long cqn,
00332                      struct arbelprm_completion_queue_context *cqctx) {
00333         return arbel_cmd ( arbel,
00334                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_HW2SW_CQ,
00335                                                1, sizeof ( *cqctx ) ),
00336                            0, NULL, cqn, cqctx );
00337 }
00338 
00339 static inline int
00340 arbel_cmd_query_cq ( struct arbel *arbel, unsigned long cqn,
00341                      struct arbelprm_completion_queue_context *cqctx ) {
00342         return arbel_cmd ( arbel,
00343                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_CQ,
00344                                                1, sizeof ( *cqctx ) ),
00345                            0, NULL, cqn, cqctx );
00346 }
00347 
00348 static inline int
00349 arbel_cmd_rst2init_qpee ( struct arbel *arbel, unsigned long qpn,
00350                           const struct arbelprm_qp_ee_state_transitions *ctx ){
00351         return arbel_cmd ( arbel,
00352                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_RST2INIT_QPEE,
00353                                               1, sizeof ( *ctx ) ),
00354                            0, ctx, qpn, NULL );
00355 }
00356 
00357 static inline int
00358 arbel_cmd_init2rtr_qpee ( struct arbel *arbel, unsigned long qpn,
00359                           const struct arbelprm_qp_ee_state_transitions *ctx ){
00360         return arbel_cmd ( arbel,
00361                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT2RTR_QPEE,
00362                                               1, sizeof ( *ctx ) ),
00363                            0, ctx, qpn, NULL );
00364 }
00365 
00366 static inline int
00367 arbel_cmd_rtr2rts_qpee ( struct arbel *arbel, unsigned long qpn,
00368                          const struct arbelprm_qp_ee_state_transitions *ctx ) {
00369         return arbel_cmd ( arbel,
00370                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_RTR2RTS_QPEE,
00371                                               1, sizeof ( *ctx ) ),
00372                            0, ctx, qpn, NULL );
00373 }
00374 
00375 static inline int
00376 arbel_cmd_rts2rts_qpee ( struct arbel *arbel, unsigned long qpn,
00377                          const struct arbelprm_qp_ee_state_transitions *ctx ) {
00378         return arbel_cmd ( arbel,
00379                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_RTS2RTS_QPEE,
00380                                               1, sizeof ( *ctx ) ),
00381                            0, ctx, qpn, NULL );
00382 }
00383 
00384 static inline int
00385 arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) {
00386         return arbel_cmd ( arbel,
00387                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_2RST_QPEE ),
00388                            0x03, NULL, qpn, NULL );
00389 }
00390 
00391 static inline int
00392 arbel_cmd_query_qpee ( struct arbel *arbel, unsigned long qpn,
00393                        struct arbelprm_qp_ee_state_transitions *ctx ) {
00394         return arbel_cmd ( arbel,
00395                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_QPEE,
00396                                                1, sizeof ( *ctx ) ),
00397                            0, NULL, qpn, ctx );
00398 }
00399 
00400 static inline int
00401 arbel_cmd_conf_special_qp ( struct arbel *arbel, unsigned int qp_type,
00402                             unsigned long base_qpn ) {
00403         return arbel_cmd ( arbel,
00404                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CONF_SPECIAL_QP ),
00405                            qp_type, NULL, base_qpn, NULL );
00406 }
00407 
00408 static inline int
00409 arbel_cmd_mad_ifc ( struct arbel *arbel, unsigned int port,
00410                     union arbelprm_mad *mad ) {
00411         return arbel_cmd ( arbel,
00412                            ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MAD_IFC,
00413                                                  1, sizeof ( *mad ),
00414                                                  1, sizeof ( *mad ) ),
00415                            0x03, mad, port, mad );
00416 }
00417 
00418 static inline int
00419 arbel_cmd_read_mgm ( struct arbel *arbel, unsigned int index,
00420                      struct arbelprm_mgm_entry *mgm ) {
00421         return arbel_cmd ( arbel,
00422                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_READ_MGM,
00423                                                1, sizeof ( *mgm ) ),
00424                            0, NULL, index, mgm );
00425 }
00426 
00427 static inline int
00428 arbel_cmd_write_mgm ( struct arbel *arbel, unsigned int index,
00429                       const struct arbelprm_mgm_entry *mgm ) {
00430         return arbel_cmd ( arbel,
00431                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_WRITE_MGM,
00432                                               1, sizeof ( *mgm ) ),
00433                            0, mgm, index, NULL );
00434 }
00435 
00436 static inline int
00437 arbel_cmd_mgid_hash ( struct arbel *arbel, const union ib_gid *gid,
00438                       struct arbelprm_mgm_hash *hash ) {
00439         return arbel_cmd ( arbel,
00440                            ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MGID_HASH,
00441                                                  1, sizeof ( *gid ),
00442                                                  0, sizeof ( *hash ) ),
00443                            0, gid, 0, hash );
00444 }
00445 
00446 static inline int
00447 arbel_cmd_run_fw ( struct arbel *arbel ) {
00448         return arbel_cmd ( arbel,
00449                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_RUN_FW ),
00450                            0, NULL, 0, NULL );
00451 }
00452 
00453 static inline int
00454 arbel_cmd_disable_lam ( struct arbel *arbel ) {
00455         return arbel_cmd ( arbel,
00456                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_DISABLE_LAM ),
00457                            0, NULL, 0, NULL );
00458 }
00459 
00460 static inline int
00461 arbel_cmd_enable_lam ( struct arbel *arbel, struct arbelprm_access_lam *lam ) {
00462         return arbel_cmd ( arbel,
00463                            ARBEL_HCR_OUT_CMD ( ARBEL_HCR_ENABLE_LAM,
00464                                                1, sizeof ( *lam ) ),
00465                            1, NULL, 0, lam );
00466 }
00467 
00468 static inline int
00469 arbel_cmd_unmap_icm ( struct arbel *arbel, unsigned int page_count,
00470                       const struct arbelprm_scalar_parameter *offset ) {
00471         return arbel_cmd ( arbel,
00472                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_UNMAP_ICM, 0,
00473                                               sizeof ( *offset ) ),
00474                            0, offset, page_count, NULL );
00475 }
00476 
00477 static inline int
00478 arbel_cmd_map_icm ( struct arbel *arbel,
00479                     const struct arbelprm_virtual_physical_mapping *map ) {
00480         return arbel_cmd ( arbel,
00481                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_MAP_ICM,
00482                                               1, sizeof ( *map ) ),
00483                            0, map, 1, NULL );
00484 }
00485 
00486 static inline int
00487 arbel_cmd_unmap_icm_aux ( struct arbel *arbel ) {
00488         return arbel_cmd ( arbel,
00489                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_UNMAP_ICM_AUX ),
00490                            0, NULL, 0, NULL );
00491 }
00492 
00493 static inline int
00494 arbel_cmd_map_icm_aux ( struct arbel *arbel,
00495                         const struct arbelprm_virtual_physical_mapping *map ) {
00496         return arbel_cmd ( arbel,
00497                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_MAP_ICM_AUX,
00498                                               1, sizeof ( *map ) ),
00499                            0, map, 1, NULL );
00500 }
00501 
00502 static inline int
00503 arbel_cmd_set_icm_size ( struct arbel *arbel,
00504                          const struct arbelprm_scalar_parameter *icm_size,
00505                          struct arbelprm_scalar_parameter *icm_aux_size ) {
00506         return arbel_cmd ( arbel,
00507                            ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_SET_ICM_SIZE,
00508                                                  0, sizeof ( *icm_size ),
00509                                                  0, sizeof ( *icm_aux_size ) ),
00510                            0, icm_size, 0, icm_aux_size );
00511 }
00512 
00513 static inline int
00514 arbel_cmd_unmap_fa ( struct arbel *arbel ) {
00515         return arbel_cmd ( arbel,
00516                            ARBEL_HCR_VOID_CMD ( ARBEL_HCR_UNMAP_FA ),
00517                            0, NULL, 0, NULL );
00518 }
00519 
00520 static inline int
00521 arbel_cmd_map_fa ( struct arbel *arbel,
00522                    const struct arbelprm_virtual_physical_mapping *map ) {
00523         return arbel_cmd ( arbel,
00524                            ARBEL_HCR_IN_CMD ( ARBEL_HCR_MAP_FA,
00525                                               1, sizeof ( *map ) ),
00526                            0, map, 1, NULL );
00527 }
00528 
00529 /***************************************************************************
00530  *
00531  * MAD operations
00532  *
00533  ***************************************************************************
00534  */
00535 
00536 /**
00537  * Issue management datagram
00538  *
00539  * @v ibdev             Infiniband device
00540  * @v mad               Management datagram
00541  * @ret rc              Return status code
00542  */
00543 static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
00544         struct arbel *arbel = ib_get_drvdata ( ibdev );
00545         union arbelprm_mad mad_ifc;
00546         int rc;
00547 
00548         linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
00549                         mad_size_mismatch );
00550 
00551         /* Copy in request packet */
00552         memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
00553 
00554         /* Issue MAD */
00555         if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
00556                                         &mad_ifc ) ) != 0 ) {
00557                 DBGC ( arbel, "Arbel %p port %d could not issue MAD IFC: %s\n",
00558                        arbel, ibdev->port, strerror ( rc ) );
00559                 return rc;
00560         }
00561 
00562         /* Copy out reply packet */
00563         memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
00564 
00565         if ( mad->hdr.status != 0 ) {
00566                 DBGC ( arbel, "Arbel %p port %d MAD IFC status %04x\n",
00567                        arbel, ibdev->port, ntohs ( mad->hdr.status ) );
00568                 return -EIO;
00569         }
00570         return 0;
00571 }
00572 
00573 /***************************************************************************
00574  *
00575  * Completion queue operations
00576  *
00577  ***************************************************************************
00578  */
00579 
00580 /**
00581  * Dump completion queue context (for debugging only)
00582  *
00583  * @v arbel             Arbel device
00584  * @v cq                Completion queue
00585  * @ret rc              Return status code
00586  */
00587 static __attribute__ (( unused )) int
00588 arbel_dump_cqctx ( struct arbel *arbel, struct ib_completion_queue *cq ) {
00589         struct arbelprm_completion_queue_context cqctx;
00590         int rc;
00591 
00592         memset ( &cqctx, 0, sizeof ( cqctx ) );
00593         if ( ( rc = arbel_cmd_query_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
00594                 DBGC ( arbel, "Arbel %p CQN %#lx QUERY_CQ failed: %s\n",
00595                        arbel, cq->cqn, strerror ( rc ) );
00596                 return rc;
00597         }
00598         DBGC ( arbel, "Arbel %p CQN %#lx context:\n", arbel, cq->cqn );
00599         DBGC_HDA ( arbel, 0, &cqctx, sizeof ( cqctx ) );
00600 
00601         return 0;
00602 }
00603 
00604 /**
00605  * Create completion queue
00606  *
00607  * @v ibdev             Infiniband device
00608  * @v cq                Completion queue
00609  * @ret rc              Return status code
00610  */
00611 static int arbel_create_cq ( struct ib_device *ibdev,
00612                              struct ib_completion_queue *cq ) {
00613         struct arbel *arbel = ib_get_drvdata ( ibdev );
00614         struct arbel_completion_queue *arbel_cq;
00615         struct arbelprm_completion_queue_context cqctx;
00616         struct arbelprm_cq_ci_db_record *ci_db_rec;
00617         struct arbelprm_cq_arm_db_record *arm_db_rec;
00618         int cqn_offset;
00619         unsigned int i;
00620         int rc;
00621 
00622         /* Find a free completion queue number */
00623         cqn_offset = arbel_bitmask_alloc ( arbel->cq_inuse, ARBEL_MAX_CQS );
00624         if ( cqn_offset < 0 ) {
00625                 DBGC ( arbel, "Arbel %p out of completion queues\n", arbel );
00626                 rc = cqn_offset;
00627                 goto err_cqn_offset;
00628         }
00629         cq->cqn = ( arbel->limits.reserved_cqs + cqn_offset );
00630 
00631         /* Allocate control structures */
00632         arbel_cq = zalloc ( sizeof ( *arbel_cq ) );
00633         if ( ! arbel_cq ) {
00634                 rc = -ENOMEM;
00635                 goto err_arbel_cq;
00636         }
00637         arbel_cq->ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( arbel, cq );
00638         arbel_cq->arm_doorbell_idx = arbel_cq_arm_doorbell_idx ( arbel, cq );
00639 
00640         /* Allocate completion queue itself */
00641         arbel_cq->cqe_size = ( cq->num_cqes * sizeof ( arbel_cq->cqe[0] ) );
00642         arbel_cq->cqe = malloc_dma ( arbel_cq->cqe_size,
00643                                      sizeof ( arbel_cq->cqe[0] ) );
00644         if ( ! arbel_cq->cqe ) {
00645                 rc = -ENOMEM;
00646                 goto err_cqe;
00647         }
00648         memset ( arbel_cq->cqe, 0, arbel_cq->cqe_size );
00649         for ( i = 0 ; i < cq->num_cqes ; i++ ) {
00650                 MLX_FILL_1 ( &arbel_cq->cqe[i].normal, 7, owner, 1 );
00651         }
00652         barrier();
00653 
00654         /* Initialise doorbell records */
00655         ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
00656         MLX_FILL_1 ( ci_db_rec, 0, counter, 0 );
00657         MLX_FILL_2 ( ci_db_rec, 1,
00658                      res, ARBEL_UAR_RES_CQ_CI,
00659                      cq_number, cq->cqn );
00660         arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
00661         MLX_FILL_1 ( arm_db_rec, 0, counter, 0 );
00662         MLX_FILL_2 ( arm_db_rec, 1,
00663                      res, ARBEL_UAR_RES_CQ_ARM,
00664                      cq_number, cq->cqn );
00665 
00666         /* Hand queue over to hardware */
00667         memset ( &cqctx, 0, sizeof ( cqctx ) );
00668         MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
00669         MLX_FILL_H ( &cqctx, 1, start_address_h,
00670                      virt_to_bus ( arbel_cq->cqe ) );
00671         MLX_FILL_1 ( &cqctx, 2, start_address_l,
00672                      virt_to_bus ( arbel_cq->cqe ) );
00673         MLX_FILL_2 ( &cqctx, 3,
00674                      usr_page, arbel->limits.reserved_uars,
00675                      log_cq_size, fls ( cq->num_cqes - 1 ) );
00676         MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eq.eqn );
00677         MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
00678         MLX_FILL_1 ( &cqctx, 7, l_key, arbel->lkey );
00679         MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
00680         MLX_FILL_1 ( &cqctx, 13,
00681                      cq_ci_db_record, arbel_cq->ci_doorbell_idx );
00682         MLX_FILL_1 ( &cqctx, 14,
00683                      cq_state_db_record, arbel_cq->arm_doorbell_idx );
00684         if ( ( rc = arbel_cmd_sw2hw_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
00685                 DBGC ( arbel, "Arbel %p CQN %#lx SW2HW_CQ failed: %s\n",
00686                        arbel, cq->cqn, strerror ( rc ) );
00687                 goto err_sw2hw_cq;
00688         }
00689 
00690         DBGC ( arbel, "Arbel %p CQN %#lx ring [%08lx,%08lx), doorbell %08lx\n",
00691                arbel, cq->cqn, virt_to_phys ( arbel_cq->cqe ),
00692                ( virt_to_phys ( arbel_cq->cqe ) + arbel_cq->cqe_size ),
00693                virt_to_phys ( ci_db_rec ) );
00694         ib_cq_set_drvdata ( cq, arbel_cq );
00695         return 0;
00696 
00697  err_sw2hw_cq:
00698         MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
00699         MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
00700         free_dma ( arbel_cq->cqe, arbel_cq->cqe_size );
00701  err_cqe:
00702         free ( arbel_cq );
00703  err_arbel_cq:
00704         arbel_bitmask_free ( arbel->cq_inuse, cqn_offset );
00705  err_cqn_offset:
00706         return rc;
00707 }
00708 
00709 /**
00710  * Destroy completion queue
00711  *
00712  * @v ibdev             Infiniband device
00713  * @v cq                Completion queue
00714  */
00715 static void arbel_destroy_cq ( struct ib_device *ibdev,
00716                                struct ib_completion_queue *cq ) {
00717         struct arbel *arbel = ib_get_drvdata ( ibdev );
00718         struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
00719         struct arbelprm_completion_queue_context cqctx;
00720         struct arbelprm_cq_ci_db_record *ci_db_rec;
00721         struct arbelprm_cq_arm_db_record *arm_db_rec;
00722         int cqn_offset;
00723         int rc;
00724 
00725         /* Take ownership back from hardware */
00726         if ( ( rc = arbel_cmd_hw2sw_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
00727                 DBGC ( arbel, "Arbel %p CQN %#lx FATAL HW2SW_CQ failed: "
00728                        "%s\n", arbel, cq->cqn, strerror ( rc ) );
00729                 /* Leak memory and return; at least we avoid corruption */
00730                 return;
00731         }
00732 
00733         /* Clear doorbell records */
00734         ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
00735         arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
00736         MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
00737         MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
00738 
00739         /* Free memory */
00740         free_dma ( arbel_cq->cqe, arbel_cq->cqe_size );
00741         free ( arbel_cq );
00742 
00743         /* Mark queue number as free */
00744         cqn_offset = ( cq->cqn - arbel->limits.reserved_cqs );
00745         arbel_bitmask_free ( arbel->cq_inuse, cqn_offset );
00746 
00747         ib_cq_set_drvdata ( cq, NULL );
00748 }
00749 
00750 /***************************************************************************
00751  *
00752  * Queue pair operations
00753  *
00754  ***************************************************************************
00755  */
00756 
00757 /**
00758  * Assign queue pair number
00759  *
00760  * @v ibdev             Infiniband device
00761  * @v qp                Queue pair
00762  * @ret rc              Return status code
00763  */
00764 static int arbel_alloc_qpn ( struct ib_device *ibdev,
00765                              struct ib_queue_pair *qp ) {
00766         struct arbel *arbel = ib_get_drvdata ( ibdev );
00767         unsigned int port_offset;
00768         int qpn_offset;
00769 
00770         /* Calculate queue pair number */
00771         port_offset = ( ibdev->port - ARBEL_PORT_BASE );
00772 
00773         switch ( qp->type ) {
00774         case IB_QPT_SMI:
00775                 qp->qpn = ( arbel->special_qpn_base + port_offset );
00776                 return 0;
00777         case IB_QPT_GSI:
00778                 qp->qpn = ( arbel->special_qpn_base + 2 + port_offset );
00779                 return 0;
00780         case IB_QPT_UD:
00781         case IB_QPT_RC:
00782                 /* Find a free queue pair number */
00783                 qpn_offset = arbel_bitmask_alloc ( arbel->qp_inuse,
00784                                                    ARBEL_MAX_QPS );
00785                 if ( qpn_offset < 0 ) {
00786                         DBGC ( arbel, "Arbel %p out of queue pairs\n",
00787                                arbel );
00788                         return qpn_offset;
00789                 }
00790                 qp->qpn = ( ( random() & ARBEL_QPN_RANDOM_MASK ) |
00791                             ( arbel->qpn_base + qpn_offset ) );
00792                 return 0;
00793         default:
00794                 DBGC ( arbel, "Arbel %p unsupported QP type %d\n",
00795                        arbel, qp->type );
00796                 return -ENOTSUP;
00797         }
00798 }
00799 
00800 /**
00801  * Free queue pair number
00802  *
00803  * @v ibdev             Infiniband device
00804  * @v qp                Queue pair
00805  */
00806 static void arbel_free_qpn ( struct ib_device *ibdev,
00807                              struct ib_queue_pair *qp ) {
00808         struct arbel *arbel = ib_get_drvdata ( ibdev );
00809         int qpn_offset;
00810 
00811         qpn_offset = ( ( qp->qpn & ~ARBEL_QPN_RANDOM_MASK ) - arbel->qpn_base );
00812         if ( qpn_offset >= 0 )
00813                 arbel_bitmask_free ( arbel->qp_inuse, qpn_offset );
00814 }
00815 
00816 /**
00817  * Calculate transmission rate
00818  *
00819  * @v av                Address vector
00820  * @ret arbel_rate      Arbel rate
00821  */
00822 static unsigned int arbel_rate ( struct ib_address_vector *av ) {
00823         return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) )
00824                  ? ( av->rate + 5 ) : 0 );
00825 }
00826 
00827 /** Queue pair transport service type map */
00828 static uint8_t arbel_qp_st[] = {
00829         [IB_QPT_SMI] = ARBEL_ST_MLX,
00830         [IB_QPT_GSI] = ARBEL_ST_MLX,
00831         [IB_QPT_UD] = ARBEL_ST_UD,
00832         [IB_QPT_RC] = ARBEL_ST_RC,
00833 };
00834 
00835 /**
00836  * Dump queue pair context (for debugging only)
00837  *
00838  * @v arbel             Arbel device
00839  * @v qp                Queue pair
00840  * @ret rc              Return status code
00841  */
00842 static __attribute__ (( unused )) int
00843 arbel_dump_qpctx ( struct arbel *arbel, struct ib_queue_pair *qp ) {
00844         struct arbelprm_qp_ee_state_transitions qpctx;
00845         int rc;
00846 
00847         memset ( &qpctx, 0, sizeof ( qpctx ) );
00848         if ( ( rc = arbel_cmd_query_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ) {
00849                 DBGC ( arbel, "Arbel %p QPN %#lx QUERY_QPEE failed: %s\n",
00850                        arbel, qp->qpn, strerror ( rc ) );
00851                 return rc;
00852         }
00853         DBGC ( arbel, "Arbel %p QPN %#lx context:\n", arbel, qp->qpn );
00854         DBGC_HDA ( arbel, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) );
00855 
00856         return 0;
00857 }
00858 
00859 /**
00860  * Create send work queue
00861  *
00862  * @v arbel_send_wq     Send work queue
00863  * @v num_wqes          Number of work queue entries
00864  * @ret rc              Return status code
00865  */
00866 static int arbel_create_send_wq ( struct arbel_send_work_queue *arbel_send_wq,
00867                                   unsigned int num_wqes ) {
00868         union arbel_send_wqe *wqe;
00869         union arbel_send_wqe *next_wqe;
00870         unsigned int wqe_idx_mask;
00871         unsigned int i;
00872 
00873         /* Allocate work queue */
00874         arbel_send_wq->wqe_size = ( num_wqes *
00875                                     sizeof ( arbel_send_wq->wqe[0] ) );
00876         arbel_send_wq->wqe = malloc_dma ( arbel_send_wq->wqe_size,
00877                                           sizeof ( arbel_send_wq->wqe[0] ) );
00878         if ( ! arbel_send_wq->wqe )
00879                 return -ENOMEM;
00880         memset ( arbel_send_wq->wqe, 0, arbel_send_wq->wqe_size );
00881 
00882         /* Link work queue entries */
00883         wqe_idx_mask = ( num_wqes - 1 );
00884         for ( i = 0 ; i < num_wqes ; i++ ) {
00885                 wqe = &arbel_send_wq->wqe[i];
00886                 next_wqe = &arbel_send_wq->wqe[ ( i + 1 ) & wqe_idx_mask ];
00887                 MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
00888                              ( virt_to_bus ( next_wqe ) >> 6 ) );
00889                 MLX_FILL_1 ( &wqe->next, 1, always1, 1 );
00890         }
00891         
00892         return 0;
00893 }
00894 
00895 /**
00896  * Create receive work queue
00897  *
00898  * @v arbel_recv_wq     Receive work queue
00899  * @v num_wqes          Number of work queue entries
00900  * @v type              Queue pair type
00901  * @ret rc              Return status code
00902  */
00903 static int arbel_create_recv_wq ( struct arbel_recv_work_queue *arbel_recv_wq,
00904                                   unsigned int num_wqes,
00905                                   enum ib_queue_pair_type type ) {
00906         struct arbelprm_recv_wqe *wqe;
00907         struct arbelprm_recv_wqe *next_wqe;
00908         unsigned int wqe_idx_mask;
00909         size_t nds;
00910         unsigned int i;
00911         unsigned int j;
00912         int rc;
00913 
00914         /* Allocate work queue */
00915         arbel_recv_wq->wqe_size = ( num_wqes *
00916                                     sizeof ( arbel_recv_wq->wqe[0] ) );
00917         arbel_recv_wq->wqe = malloc_dma ( arbel_recv_wq->wqe_size,
00918                                           sizeof ( arbel_recv_wq->wqe[0] ) );
00919         if ( ! arbel_recv_wq->wqe ) {
00920                 rc = -ENOMEM;
00921                 goto err_alloc_wqe;
00922         }
00923         memset ( arbel_recv_wq->wqe, 0, arbel_recv_wq->wqe_size );
00924 
00925         /* Allocate GRH entries, if needed */
00926         if ( ( type == IB_QPT_SMI ) || ( type == IB_QPT_GSI ) ||
00927              ( type == IB_QPT_UD ) ) {
00928                 arbel_recv_wq->grh_size = ( num_wqes *
00929                                             sizeof ( arbel_recv_wq->grh[0] ) );
00930                 arbel_recv_wq->grh = malloc_dma ( arbel_recv_wq->grh_size,
00931                                                   sizeof ( void * ) );
00932                 if ( ! arbel_recv_wq->grh ) {
00933                         rc = -ENOMEM;
00934                         goto err_alloc_grh;
00935                 }
00936         }
00937 
00938         /* Link work queue entries */
00939         wqe_idx_mask = ( num_wqes - 1 );
00940         nds = ( ( offsetof ( typeof ( *wqe ), data ) +
00941                   sizeof ( wqe->data[0] ) ) >> 4 );
00942         for ( i = 0 ; i < num_wqes ; i++ ) {
00943                 wqe = &arbel_recv_wq->wqe[i].recv;
00944                 next_wqe = &arbel_recv_wq->wqe[( i + 1 ) & wqe_idx_mask].recv;
00945                 MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
00946                              ( virt_to_bus ( next_wqe ) >> 6 ) );
00947                 MLX_FILL_1 ( &wqe->next, 1, nds, nds );
00948                 for ( j = 0 ; ( ( ( void * ) &wqe->data[j] ) <
00949                                 ( ( void * ) ( wqe + 1 ) ) ) ; j++ ) {
00950                         MLX_FILL_1 ( &wqe->data[j], 1,
00951                                      l_key, ARBEL_INVALID_LKEY );
00952                 }
00953         }
00954         
00955         return 0;
00956 
00957         free_dma ( arbel_recv_wq->grh, arbel_recv_wq->grh_size );
00958  err_alloc_grh:
00959         free_dma ( arbel_recv_wq->wqe, arbel_recv_wq->wqe_size );
00960  err_alloc_wqe:
00961         return rc;
00962 }
00963 
00964 /**
00965  * Create queue pair
00966  *
00967  * @v ibdev             Infiniband device
00968  * @v qp                Queue pair
00969  * @ret rc              Return status code
00970  */
00971 static int arbel_create_qp ( struct ib_device *ibdev,
00972                              struct ib_queue_pair *qp ) {
00973         struct arbel *arbel = ib_get_drvdata ( ibdev );
00974         struct arbel_queue_pair *arbel_qp;
00975         struct arbelprm_qp_ee_state_transitions qpctx;
00976         struct arbelprm_qp_db_record *send_db_rec;
00977         struct arbelprm_qp_db_record *recv_db_rec;
00978         physaddr_t send_wqe_base_adr;
00979         physaddr_t recv_wqe_base_adr;
00980         physaddr_t wqe_base_adr;
00981         int rc;
00982 
00983         /* Warn about dysfunctional code
00984          *
00985          * Arbel seems to crash the system as soon as the first send
00986          * WQE completes on an RC queue pair.  (NOPs complete
00987          * successfully, so this is a problem specific to the work
00988          * queue rather than the completion queue.)  The cause of this
00989          * problem has remained unknown for over a year.  Patches to
00990          * fix this are welcome.
00991          */
00992         if ( qp->type == IB_QPT_RC )
00993                 DBG ( "*** WARNING: Arbel RC support is non-functional ***\n" );
00994 
00995         /* Calculate queue pair number */
00996         if ( ( rc = arbel_alloc_qpn ( ibdev, qp ) ) != 0 )
00997                 goto err_alloc_qpn;
00998 
00999         /* Allocate control structures */
01000         arbel_qp = zalloc ( sizeof ( *arbel_qp ) );
01001         if ( ! arbel_qp ) {
01002                 rc = -ENOMEM;
01003                 goto err_arbel_qp;
01004         }
01005         arbel_qp->send.doorbell_idx = arbel_send_doorbell_idx ( arbel, qp );
01006         arbel_qp->recv.doorbell_idx = arbel_recv_doorbell_idx ( arbel, qp );
01007 
01008         /* Create send and receive work queues */
01009         if ( ( rc = arbel_create_send_wq ( &arbel_qp->send,
01010                                            qp->send.num_wqes ) ) != 0 )
01011                 goto err_create_send_wq;
01012         if ( ( rc = arbel_create_recv_wq ( &arbel_qp->recv, qp->recv.num_wqes,
01013                                            qp->type ) ) != 0 )
01014                 goto err_create_recv_wq;
01015 
01016         /* Send and receive work queue entries must be within the same 4GB */
01017         send_wqe_base_adr = virt_to_bus ( arbel_qp->send.wqe );
01018         recv_wqe_base_adr = virt_to_bus ( arbel_qp->recv.wqe );
01019         if ( ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) &&
01020              ( ( ( ( uint64_t ) send_wqe_base_adr ) >> 32 ) !=
01021                ( ( ( uint64_t ) recv_wqe_base_adr ) >> 32 ) ) ) {
01022                 DBGC ( arbel, "Arbel %p QPN %#lx cannot support send %08lx "
01023                        "recv %08lx\n", arbel, qp->qpn,
01024                        send_wqe_base_adr, recv_wqe_base_adr );
01025                 rc = -ENOTSUP;
01026                 goto err_unsupported_address_split;
01027         }
01028         wqe_base_adr = send_wqe_base_adr;
01029 
01030         /* Initialise doorbell records */
01031         send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
01032         MLX_FILL_1 ( send_db_rec, 0, counter, 0 );
01033         MLX_FILL_2 ( send_db_rec, 1,
01034                      res, ARBEL_UAR_RES_SQ,
01035                      qp_number, qp->qpn );
01036         recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
01037         MLX_FILL_1 ( recv_db_rec, 0, counter, 0 );
01038         MLX_FILL_2 ( recv_db_rec, 1,
01039                      res, ARBEL_UAR_RES_RQ,
01040                      qp_number, qp->qpn );
01041 
01042         /* Transition queue to INIT state */
01043         memset ( &qpctx, 0, sizeof ( qpctx ) );
01044         MLX_FILL_3 ( &qpctx, 2,
01045                      qpc_eec_data.de, 1,
01046                      qpc_eec_data.pm_state, ARBEL_PM_STATE_MIGRATED,
01047                      qpc_eec_data.st, arbel_qp_st[qp->type] );
01048         MLX_FILL_4 ( &qpctx, 4,
01049                      qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
01050                      qpc_eec_data.log_rq_stride,
01051                      ( fls ( sizeof ( arbel_qp->recv.wqe[0] ) - 1 ) - 4 ),
01052                      qpc_eec_data.log_sq_size, fls ( qp->send.num_wqes - 1 ),
01053                      qpc_eec_data.log_sq_stride,
01054                      ( fls ( sizeof ( arbel_qp->send.wqe[0] ) - 1 ) - 4 ) );
01055         MLX_FILL_1 ( &qpctx, 5,
01056                      qpc_eec_data.usr_page, arbel->limits.reserved_uars );
01057         MLX_FILL_1 ( &qpctx, 10, qpc_eec_data.primary_address_path.port_number,
01058                      ibdev->port );
01059         MLX_FILL_1 ( &qpctx, 27, qpc_eec_data.pd, ARBEL_GLOBAL_PD );
01060         MLX_FILL_H ( &qpctx, 28, qpc_eec_data.wqe_base_adr_h, wqe_base_adr );
01061         MLX_FILL_1 ( &qpctx, 29, qpc_eec_data.wqe_lkey, arbel->lkey );
01062         MLX_FILL_1 ( &qpctx, 30, qpc_eec_data.ssc, 1 );
01063         MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
01064         MLX_FILL_1 ( &qpctx, 34, qpc_eec_data.snd_wqe_base_adr_l,
01065                      ( send_wqe_base_adr >> 6 ) );
01066         MLX_FILL_1 ( &qpctx, 35, qpc_eec_data.snd_db_record_index,
01067                      arbel_qp->send.doorbell_idx );
01068         MLX_FILL_4 ( &qpctx, 38,
01069                      qpc_eec_data.rre, 1,
01070                      qpc_eec_data.rwe, 1,
01071                      qpc_eec_data.rae, 1,
01072                      qpc_eec_data.rsc, 1 );
01073         MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
01074         MLX_FILL_1 ( &qpctx, 42, qpc_eec_data.rcv_wqe_base_adr_l,
01075                      ( recv_wqe_base_adr >> 6 ) );
01076         MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.rcv_db_record_index,
01077                      arbel_qp->recv.doorbell_idx );
01078         if ( ( rc = arbel_cmd_rst2init_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
01079                 DBGC ( arbel, "Arbel %p QPN %#lx RST2INIT_QPEE failed: %s\n",
01080                        arbel, qp->qpn, strerror ( rc ) );
01081                 goto err_rst2init_qpee;
01082         }
01083         arbel_qp->state = ARBEL_QP_ST_INIT;
01084 
01085         DBGC ( arbel, "Arbel %p QPN %#lx send ring [%08lx,%08lx), doorbell "
01086                "%08lx\n", arbel, qp->qpn, virt_to_phys ( arbel_qp->send.wqe ),
01087                ( virt_to_phys ( arbel_qp->send.wqe ) +
01088                  arbel_qp->send.wqe_size ),
01089                virt_to_phys ( send_db_rec ) );
01090         DBGC ( arbel, "Arbel %p QPN %#lx receive ring [%08lx,%08lx), doorbell "
01091                "%08lx\n", arbel, qp->qpn, virt_to_phys ( arbel_qp->recv.wqe ),
01092                ( virt_to_phys ( arbel_qp->recv.wqe ) +
01093                  arbel_qp->recv.wqe_size ),
01094                virt_to_phys ( recv_db_rec ) );
01095         DBGC ( arbel, "Arbel %p QPN %#lx send CQN %#lx receive CQN %#lx\n",
01096                arbel, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn );
01097         ib_qp_set_drvdata ( qp, arbel_qp );
01098         return 0;
01099 
01100         arbel_cmd_2rst_qpee ( arbel, qp->qpn );
01101  err_rst2init_qpee:
01102         MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
01103         MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
01104  err_unsupported_address_split:
01105         free_dma ( arbel_qp->recv.grh, arbel_qp->recv.grh_size );
01106         free_dma ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
01107  err_create_recv_wq:
01108         free_dma ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
01109  err_create_send_wq:
01110         free ( arbel_qp );
01111  err_arbel_qp:
01112         arbel_free_qpn ( ibdev, qp );
01113  err_alloc_qpn:
01114         return rc;
01115 }
01116 
01117 /**
01118  * Modify queue pair
01119  *
01120  * @v ibdev             Infiniband device
01121  * @v qp                Queue pair
01122  * @ret rc              Return status code
01123  */
01124 static int arbel_modify_qp ( struct ib_device *ibdev,
01125                              struct ib_queue_pair *qp ) {
01126         struct arbel *arbel = ib_get_drvdata ( ibdev );
01127         struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
01128         struct arbelprm_qp_ee_state_transitions qpctx;
01129         int rc;
01130 
01131         /* Transition queue to RTR state, if applicable */
01132         if ( arbel_qp->state < ARBEL_QP_ST_RTR ) {
01133                 memset ( &qpctx, 0, sizeof ( qpctx ) );
01134                 MLX_FILL_2 ( &qpctx, 4,
01135                              qpc_eec_data.mtu, ARBEL_MTU_2048,
01136                              qpc_eec_data.msg_max, 31 );
01137                 MLX_FILL_1 ( &qpctx, 7,
01138                              qpc_eec_data.remote_qpn_een, qp->av.qpn );
01139                 MLX_FILL_2 ( &qpctx, 11,
01140                              qpc_eec_data.primary_address_path.rnr_retry,
01141                              ARBEL_RETRY_MAX,
01142                              qpc_eec_data.primary_address_path.rlid,
01143                              qp->av.lid );
01144                 MLX_FILL_2 ( &qpctx, 12,
01145                              qpc_eec_data.primary_address_path.ack_timeout,
01146                              14 /* 4.096us * 2^(14) = 67ms */,
01147                              qpc_eec_data.primary_address_path.max_stat_rate,
01148                              arbel_rate ( &qp->av ) );
01149                 memcpy ( &qpctx.u.dwords[14], &qp->av.gid,
01150                          sizeof ( qp->av.gid ) );
01151                 MLX_FILL_1 ( &qpctx, 30,
01152                              qpc_eec_data.retry_count, ARBEL_RETRY_MAX );
01153                 MLX_FILL_1 ( &qpctx, 39,
01154                              qpc_eec_data.next_rcv_psn, qp->recv.psn );
01155                 MLX_FILL_1 ( &qpctx, 40,
01156                              qpc_eec_data.ra_buff_indx,
01157                              ( arbel->limits.reserved_rdbs +
01158                                ( ( qp->qpn & ~ARBEL_QPN_RANDOM_MASK ) -
01159                                  arbel->special_qpn_base ) ) );
01160                 if ( ( rc = arbel_cmd_init2rtr_qpee ( arbel, qp->qpn,
01161                                                       &qpctx ) ) != 0 ) {
01162                         DBGC ( arbel, "Arbel %p QPN %#lx INIT2RTR_QPEE failed:"
01163                                " %s\n", arbel, qp->qpn, strerror ( rc ) );
01164                         return rc;
01165                 }
01166                 arbel_qp->state = ARBEL_QP_ST_RTR;
01167         }
01168 
01169         /* Transition queue to RTS state, if applicable */
01170         if ( arbel_qp->state < ARBEL_QP_ST_RTS ) {
01171                 memset ( &qpctx, 0, sizeof ( qpctx ) );
01172                 MLX_FILL_1 ( &qpctx, 11,
01173                              qpc_eec_data.primary_address_path.rnr_retry,
01174                              ARBEL_RETRY_MAX );
01175                 MLX_FILL_1 ( &qpctx, 12,
01176                              qpc_eec_data.primary_address_path.ack_timeout,
01177                              14 /* 4.096us * 2^(14) = 67ms */ );
01178                 MLX_FILL_2 ( &qpctx, 30,
01179                              qpc_eec_data.retry_count, ARBEL_RETRY_MAX,
01180                              qpc_eec_data.sic, 1 );
01181                 MLX_FILL_1 ( &qpctx, 32,
01182                              qpc_eec_data.next_send_psn, qp->send.psn );
01183                 if ( ( rc = arbel_cmd_rtr2rts_qpee ( arbel, qp->qpn,
01184                                                      &qpctx ) ) != 0 ) {
01185                         DBGC ( arbel, "Arbel %p QPN %#lx RTR2RTS_QPEE failed: "
01186                                "%s\n", arbel, qp->qpn, strerror ( rc ) );
01187                         return rc;
01188                 }
01189                 arbel_qp->state = ARBEL_QP_ST_RTS;
01190         }
01191 
01192         /* Update parameters in RTS state */
01193         memset ( &qpctx, 0, sizeof ( qpctx ) );
01194         MLX_FILL_1 ( &qpctx, 0, opt_param_mask, ARBEL_QPEE_OPT_PARAM_QKEY );
01195         MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
01196         if ( ( rc = arbel_cmd_rts2rts_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ){
01197                 DBGC ( arbel, "Arbel %p QPN %#lx RTS2RTS_QPEE failed: %s\n",
01198                        arbel, qp->qpn, strerror ( rc ) );
01199                 return rc;
01200         }
01201 
01202         return 0;
01203 }
01204 
01205 /**
01206  * Destroy queue pair
01207  *
01208  * @v ibdev             Infiniband device
01209  * @v qp                Queue pair
01210  */
01211 static void arbel_destroy_qp ( struct ib_device *ibdev,
01212                                struct ib_queue_pair *qp ) {
01213         struct arbel *arbel = ib_get_drvdata ( ibdev );
01214         struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
01215         struct arbelprm_qp_db_record *send_db_rec;
01216         struct arbelprm_qp_db_record *recv_db_rec;
01217         int rc;
01218 
01219         /* Take ownership back from hardware */
01220         if ( ( rc = arbel_cmd_2rst_qpee ( arbel, qp->qpn ) ) != 0 ) {
01221                 DBGC ( arbel, "Arbel %p QPN %#lx FATAL 2RST_QPEE failed: "
01222                        "%s\n", arbel, qp->qpn, strerror ( rc ) );
01223                 /* Leak memory and return; at least we avoid corruption */
01224                 return;
01225         }
01226 
01227         /* Clear doorbell records */
01228         send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
01229         recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
01230         MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
01231         MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
01232 
01233         /* Free memory */
01234         free_dma ( arbel_qp->recv.grh, arbel_qp->recv.grh_size );
01235         free_dma ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
01236         free_dma ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
01237         free ( arbel_qp );
01238 
01239         /* Mark queue number as free */
01240         arbel_free_qpn ( ibdev, qp );
01241 
01242         ib_qp_set_drvdata ( qp, NULL );
01243 }
01244 
01245 /***************************************************************************
01246  *
01247  * Work request operations
01248  *
01249  ***************************************************************************
01250  */
01251 
01252 /**
01253  * Ring doorbell register in UAR
01254  *
01255  * @v arbel             Arbel device
01256  * @v db_reg            Doorbell register structure
01257  * @v offset            Address of doorbell
01258  */
01259 static void arbel_ring_doorbell ( struct arbel *arbel,
01260                                   union arbelprm_doorbell_register *db_reg,
01261                                   unsigned int offset ) {
01262 
01263         DBGC2 ( arbel, "Arbel %p ringing doorbell %08x:%08x at %lx\n",
01264                 arbel, ntohl ( db_reg->dword[0] ), ntohl ( db_reg->dword[1] ),
01265                 virt_to_phys ( arbel->uar + offset ) );
01266 
01267         barrier();
01268         writel ( db_reg->dword[0], ( arbel->uar + offset + 0 ) );
01269         barrier();
01270         writel ( db_reg->dword[1], ( arbel->uar + offset + 4 ) );
01271 }
01272 
01273 /** GID used for GID-less send work queue entries */
01274 static const union ib_gid arbel_no_gid = {
01275         .bytes = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0 },
01276 };
01277 
01278 /**
01279  * Construct UD send work queue entry
01280  *
01281  * @v ibdev             Infiniband device
01282  * @v qp                Queue pair
01283  * @v dest              Destination address vector
01284  * @v iobuf             I/O buffer
01285  * @v wqe               Send work queue entry
01286  * @ret nds             Work queue entry size
01287  */
01288 static size_t arbel_fill_ud_send_wqe ( struct ib_device *ibdev,
01289                                        struct ib_queue_pair *qp __unused,
01290                                        struct ib_address_vector *dest,
01291                                        struct io_buffer *iobuf,
01292                                        union arbel_send_wqe *wqe ) {
01293         struct arbel *arbel = ib_get_drvdata ( ibdev );
01294         const union ib_gid *gid;
01295 
01296         /* Construct this work queue entry */
01297         MLX_FILL_1 ( &wqe->ud.ctrl, 0, always1, 1 );
01298         MLX_FILL_2 ( &wqe->ud.ud, 0,
01299                      ud_address_vector.pd, ARBEL_GLOBAL_PD,
01300                      ud_address_vector.port_number, ibdev->port );
01301         MLX_FILL_2 ( &wqe->ud.ud, 1,
01302                      ud_address_vector.rlid, dest->lid,
01303                      ud_address_vector.g, dest->gid_present );
01304         MLX_FILL_2 ( &wqe->ud.ud, 2,
01305                      ud_address_vector.max_stat_rate, arbel_rate ( dest ),
01306                      ud_address_vector.msg, 3 );
01307         MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, dest->sl );
01308         gid = ( dest->gid_present ? &dest->gid : &arbel_no_gid );
01309         memcpy ( &wqe->ud.ud.u.dwords[4], gid, sizeof ( *gid ) );
01310         MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, dest->qpn );
01311         MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, dest->qkey );
01312         MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) );
01313         MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, arbel->lkey );
01314         MLX_FILL_H ( &wqe->ud.data[0], 2,
01315                      local_address_h, virt_to_bus ( iobuf->data ) );
01316         MLX_FILL_1 ( &wqe->ud.data[0], 3,
01317                      local_address_l, virt_to_bus ( iobuf->data ) );
01318 
01319         return ( offsetof ( typeof ( wqe->ud ), data[1] ) >> 4 );
01320 }
01321 
01322 /**
01323  * Construct MLX send work queue entry
01324  *
01325  * @v ibdev             Infiniband device
01326  * @v qp                Queue pair
01327  * @v dest              Destination address vector
01328  * @v iobuf             I/O buffer
01329  * @v wqe               Send work queue entry
01330  * @ret nds             Work queue entry size
01331  */
01332 static size_t arbel_fill_mlx_send_wqe ( struct ib_device *ibdev,
01333                                         struct ib_queue_pair *qp,
01334                                         struct ib_address_vector *dest,
01335                                         struct io_buffer *iobuf,
01336                                         union arbel_send_wqe *wqe ) {
01337         struct arbel *arbel = ib_get_drvdata ( ibdev );
01338         struct io_buffer headers;
01339 
01340         /* Construct IB headers */
01341         iob_populate ( &headers, &wqe->mlx.headers, 0,
01342                        sizeof ( wqe->mlx.headers ) );
01343         iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) );
01344         ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
01345 
01346         /* Construct this work queue entry */
01347         MLX_FILL_5 ( &wqe->mlx.ctrl, 0,
01348                      c, 1 /* generate completion */,
01349                      icrc, 0 /* generate ICRC */,
01350                      max_statrate, arbel_rate ( dest ),
01351                      slr, 0,
01352                      v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) );
01353         MLX_FILL_1 ( &wqe->mlx.ctrl, 1, rlid, dest->lid );
01354         MLX_FILL_1 ( &wqe->mlx.data[0], 0,
01355                      byte_count, iob_len ( &headers ) );
01356         MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, arbel->lkey );
01357         MLX_FILL_H ( &wqe->mlx.data[0], 2,
01358                      local_address_h, virt_to_bus ( headers.data ) );
01359         MLX_FILL_1 ( &wqe->mlx.data[0], 3,
01360                      local_address_l, virt_to_bus ( headers.data ) );
01361         MLX_FILL_1 ( &wqe->mlx.data[1], 0,
01362                      byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) );
01363         MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, arbel->lkey );
01364         MLX_FILL_H ( &wqe->mlx.data[1], 2,
01365                      local_address_h, virt_to_bus ( iobuf->data ) );
01366         MLX_FILL_1 ( &wqe->mlx.data[1], 3,
01367                      local_address_l, virt_to_bus ( iobuf->data ) );
01368 
01369         return ( offsetof ( typeof ( wqe->mlx ), data[2] ) >> 4 );
01370 }
01371 
01372 /**
01373  * Construct RC send work queue entry
01374  *
01375  * @v ibdev             Infiniband device
01376  * @v qp                Queue pair
01377  * @v dest              Destination address vector
01378  * @v iobuf             I/O buffer
01379  * @v wqe               Send work queue entry
01380  * @ret nds             Work queue entry size
01381  */
01382 static size_t arbel_fill_rc_send_wqe ( struct ib_device *ibdev,
01383                                        struct ib_queue_pair *qp __unused,
01384                                        struct ib_address_vector *dest __unused,
01385                                        struct io_buffer *iobuf,
01386                                        union arbel_send_wqe *wqe ) {
01387         struct arbel *arbel = ib_get_drvdata ( ibdev );
01388 
01389         /* Construct this work queue entry */
01390         MLX_FILL_1 ( &wqe->rc.ctrl, 0, always1, 1 );
01391         MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) );
01392         MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, arbel->lkey );
01393         MLX_FILL_H ( &wqe->rc.data[0], 2,
01394                      local_address_h, virt_to_bus ( iobuf->data ) );
01395         MLX_FILL_1 ( &wqe->rc.data[0], 3,
01396                      local_address_l, virt_to_bus ( iobuf->data ) );
01397 
01398         return ( offsetof ( typeof ( wqe->rc ), data[1] ) >> 4 );
01399 }
01400 
01401 /** Work queue entry constructors */
01402 static size_t
01403 ( * arbel_fill_send_wqe[] ) ( struct ib_device *ibdev,
01404                               struct ib_queue_pair *qp,
01405                               struct ib_address_vector *dest,
01406                               struct io_buffer *iobuf,
01407                               union arbel_send_wqe *wqe ) = {
01408         [IB_QPT_SMI] = arbel_fill_mlx_send_wqe,
01409         [IB_QPT_GSI] = arbel_fill_mlx_send_wqe,
01410         [IB_QPT_UD] = arbel_fill_ud_send_wqe,
01411         [IB_QPT_RC] = arbel_fill_rc_send_wqe,
01412 };
01413 
01414 /**
01415  * Post send work queue entry
01416  *
01417  * @v ibdev             Infiniband device
01418  * @v qp                Queue pair
01419  * @v dest              Destination address vector
01420  * @v iobuf             I/O buffer
01421  * @ret rc              Return status code
01422  */
01423 static int arbel_post_send ( struct ib_device *ibdev,
01424                              struct ib_queue_pair *qp,
01425                              struct ib_address_vector *dest,
01426                              struct io_buffer *iobuf ) {
01427         struct arbel *arbel = ib_get_drvdata ( ibdev );
01428         struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
01429         struct ib_work_queue *wq = &qp->send;
01430         struct arbel_send_work_queue *arbel_send_wq = &arbel_qp->send;
01431         union arbel_send_wqe *prev_wqe;
01432         union arbel_send_wqe *wqe;
01433         struct arbelprm_qp_db_record *qp_db_rec;
01434         union arbelprm_doorbell_register db_reg;
01435         unsigned long wqe_idx_mask;
01436         size_t nds;
01437 
01438         /* Allocate work queue entry */
01439         wqe_idx_mask = ( wq->num_wqes - 1 );
01440         if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
01441                 DBGC ( arbel, "Arbel %p QPN %#lx send queue full",
01442                        arbel, qp->qpn );
01443                 return -ENOBUFS;
01444         }
01445         wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
01446         prev_wqe = &arbel_send_wq->wqe[(wq->next_idx - 1) & wqe_idx_mask];
01447         wqe = &arbel_send_wq->wqe[wq->next_idx & wqe_idx_mask];
01448 
01449         /* Construct work queue entry */
01450         memset ( ( ( ( void * ) wqe ) + sizeof ( wqe->next ) ), 0,
01451                  ( sizeof ( *wqe ) - sizeof ( wqe->next ) ) );
01452         assert ( qp->type < ( sizeof ( arbel_fill_send_wqe ) /
01453                               sizeof ( arbel_fill_send_wqe[0] ) ) );
01454         assert ( arbel_fill_send_wqe[qp->type] != NULL );
01455         nds = arbel_fill_send_wqe[qp->type] ( ibdev, qp, dest, iobuf, wqe );
01456         DBGCP ( arbel, "Arbel %p QPN %#lx posting send WQE %#lx:\n",
01457                 arbel, qp->qpn, ( wq->next_idx & wqe_idx_mask ) );
01458         DBGCP_HDA ( arbel, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) );
01459 
01460         /* Update previous work queue entry's "next" field */
01461         MLX_SET ( &prev_wqe->next, nopcode, ARBEL_OPCODE_SEND );
01462         MLX_FILL_3 ( &prev_wqe->next, 1,
01463                      nds, nds,
01464                      f, 0,
01465                      always1, 1 );
01466 
01467         /* Update doorbell record */
01468         barrier();
01469         qp_db_rec = &arbel->db_rec[arbel_send_wq->doorbell_idx].qp;
01470         MLX_FILL_1 ( qp_db_rec, 0,
01471                      counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
01472 
01473         /* Ring doorbell register */
01474         MLX_FILL_4 ( &db_reg.send, 0,
01475                      nopcode, ARBEL_OPCODE_SEND,
01476                      f, 0,
01477                      wqe_counter, ( wq->next_idx & 0xffff ),
01478                      wqe_cnt, 1 );
01479         MLX_FILL_2 ( &db_reg.send, 1,
01480                      nds, nds,
01481                      qpn, qp->qpn );
01482         arbel_ring_doorbell ( arbel, &db_reg, ARBEL_DB_POST_SND_OFFSET );
01483 
01484         /* Update work queue's index */
01485         wq->next_idx++;
01486 
01487         return 0;
01488 }
01489 
01490 /**
01491  * Post receive work queue entry
01492  *
01493  * @v ibdev             Infiniband device
01494  * @v qp                Queue pair
01495  * @v iobuf             I/O buffer
01496  * @ret rc              Return status code
01497  */
01498 static int arbel_post_recv ( struct ib_device *ibdev,
01499                              struct ib_queue_pair *qp,
01500                              struct io_buffer *iobuf ) {
01501         struct arbel *arbel = ib_get_drvdata ( ibdev );
01502         struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
01503         struct ib_work_queue *wq = &qp->recv;
01504         struct arbel_recv_work_queue *arbel_recv_wq = &arbel_qp->recv;
01505         struct arbelprm_recv_wqe *wqe;
01506         struct arbelprm_wqe_segment_data_ptr *data;
01507         struct ib_global_route_header *grh;
01508         union arbelprm_doorbell_record *db_rec;
01509         unsigned int wqe_idx_mask;
01510 
01511         /* Allocate work queue entry */
01512         wqe_idx_mask = ( wq->num_wqes - 1 );
01513         if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
01514                 DBGC ( arbel, "Arbel %p QPN %#lx receive queue full\n",
01515                        arbel, qp->qpn );
01516                 return -ENOBUFS;
01517         }
01518         wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
01519         wqe = &arbel_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
01520 
01521         /* Construct work queue entry */
01522         data = &wqe->data[0];
01523         if ( arbel_recv_wq->grh ) {
01524                 grh = &arbel_recv_wq->grh[wq->next_idx & wqe_idx_mask];
01525                 MLX_FILL_1 ( data, 0, byte_count, sizeof ( *grh ) );
01526                 MLX_FILL_1 ( data, 1, l_key, arbel->lkey );
01527                 MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( grh ) );
01528                 MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( grh ) );
01529                 data++;
01530         }
01531         MLX_FILL_1 ( data, 0, byte_count, iob_tailroom ( iobuf ) );
01532         MLX_FILL_1 ( data, 1, l_key, arbel->lkey );
01533         MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( iobuf->data ) );
01534         MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( iobuf->data ) );
01535 
01536         /* Update doorbell record */
01537         barrier();
01538         db_rec = &arbel->db_rec[arbel_recv_wq->doorbell_idx];
01539         MLX_FILL_1 ( &db_rec->qp, 0,
01540                      counter, ( ( wq->next_idx + 1 ) & 0xffff ) );      
01541 
01542         /* Update work queue's index */
01543         wq->next_idx++;
01544 
01545         return 0;       
01546 }
01547 
01548 /**
01549  * Handle completion
01550  *
01551  * @v ibdev             Infiniband device
01552  * @v cq                Completion queue
01553  * @v cqe               Hardware completion queue entry
01554  * @ret rc              Return status code
01555  */
01556 static int arbel_complete ( struct ib_device *ibdev,
01557                             struct ib_completion_queue *cq,
01558                             union arbelprm_completion_entry *cqe ) {
01559         struct arbel *arbel = ib_get_drvdata ( ibdev );
01560         struct ib_work_queue *wq;
01561         struct ib_queue_pair *qp;
01562         struct arbel_queue_pair *arbel_qp;
01563         struct arbel_send_work_queue *arbel_send_wq;
01564         struct arbel_recv_work_queue *arbel_recv_wq;
01565         struct arbelprm_recv_wqe *recv_wqe;
01566         struct io_buffer *iobuf;
01567         struct ib_address_vector recv_dest;
01568         struct ib_address_vector recv_source;
01569         struct ib_global_route_header *grh;
01570         struct ib_address_vector *source;
01571         unsigned int opcode;
01572         unsigned long qpn;
01573         int is_send;
01574         unsigned long wqe_adr;
01575         unsigned long wqe_idx;
01576         size_t len;
01577         int rc = 0;
01578 
01579         /* Parse completion */
01580         qpn = MLX_GET ( &cqe->normal, my_qpn );
01581         is_send = MLX_GET ( &cqe->normal, s );
01582         wqe_adr = ( MLX_GET ( &cqe->normal, wqe_adr ) << 6 );
01583         opcode = MLX_GET ( &cqe->normal, opcode );
01584         if ( opcode >= ARBEL_OPCODE_RECV_ERROR ) {
01585                 /* "s" field is not valid for error opcodes */
01586                 is_send = ( opcode == ARBEL_OPCODE_SEND_ERROR );
01587                 DBGC ( arbel, "Arbel %p CQN %#lx %s QPN %#lx syndrome %#x "
01588                        "vendor %#x\n", arbel, cq->cqn,
01589                        ( is_send ? "send" : "recv" ), qpn,
01590                        MLX_GET ( &cqe->error, syndrome ),
01591                        MLX_GET ( &cqe->error, vendor_code ) );
01592                 DBGC_HDA ( arbel, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
01593                 rc = -EIO;
01594                 /* Don't return immediately; propagate error to completer */
01595         }
01596 
01597         /* Identify work queue */
01598         wq = ib_find_wq ( cq, qpn, is_send );
01599         if ( ! wq ) {
01600                 DBGC ( arbel, "Arbel %p CQN %#lx unknown %s QPN %#lx\n",
01601                        arbel, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
01602                 return -EIO;
01603         }
01604         qp = wq->qp;
01605         arbel_qp = ib_qp_get_drvdata ( qp );
01606         arbel_send_wq = &arbel_qp->send;
01607         arbel_recv_wq = &arbel_qp->recv;
01608 
01609         /* Identify work queue entry index */
01610         if ( is_send ) {
01611                 wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_send_wq->wqe ) ) /
01612                             sizeof ( arbel_send_wq->wqe[0] ) );
01613                 assert ( wqe_idx < qp->send.num_wqes );
01614         } else {
01615                 wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_recv_wq->wqe ) ) /
01616                             sizeof ( arbel_recv_wq->wqe[0] ) );
01617                 assert ( wqe_idx < qp->recv.num_wqes );
01618         }
01619 
01620         DBGCP ( arbel, "Arbel %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n",
01621                 arbel, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
01622                 wqe_idx );
01623         DBGCP_HDA ( arbel, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
01624 
01625         /* Identify I/O buffer */
01626         iobuf = wq->iobufs[wqe_idx];
01627         if ( ! iobuf ) {
01628                 DBGC ( arbel, "Arbel %p CQN %#lx QPN %#lx empty %s WQE %#lx\n",
01629                        arbel, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
01630                        wqe_idx );
01631                 return -EIO;
01632         }
01633         wq->iobufs[wqe_idx] = NULL;
01634 
01635         if ( is_send ) {
01636                 /* Hand off to completion handler */
01637                 ib_complete_send ( ibdev, qp, iobuf, rc );
01638         } else {
01639                 /* Set received length */
01640                 len = MLX_GET ( &cqe->normal, byte_cnt );
01641                 recv_wqe = &arbel_recv_wq->wqe[wqe_idx].recv;
01642                 assert ( MLX_GET ( &recv_wqe->data[0], local_address_l ) ==
01643                          virt_to_bus ( iobuf->data ) );
01644                 assert ( MLX_GET ( &recv_wqe->data[0], byte_count ) ==
01645                          iob_tailroom ( iobuf ) );
01646                 MLX_FILL_1 ( &recv_wqe->data[0], 0, byte_count, 0 );
01647                 MLX_FILL_1 ( &recv_wqe->data[0], 1,
01648                              l_key, ARBEL_INVALID_LKEY );
01649                 memset ( &recv_dest, 0, sizeof ( recv_dest ) );
01650                 recv_dest.qpn = qpn;
01651                 switch ( qp->type ) {
01652                 case IB_QPT_SMI:
01653                 case IB_QPT_GSI:
01654                 case IB_QPT_UD:
01655                         /* Locate corresponding GRH */
01656                         assert ( arbel_recv_wq->grh != NULL );
01657                         grh = &arbel_recv_wq->grh[wqe_idx];
01658                         len -= sizeof ( *grh );
01659                         /* Construct address vector */
01660                         source = &recv_source;
01661                         memset ( source, 0, sizeof ( *source ) );
01662                         source->qpn = MLX_GET ( &cqe->normal, rqpn );
01663                         source->lid = MLX_GET ( &cqe->normal, rlid );
01664                         source->sl = MLX_GET ( &cqe->normal, sl );
01665                         recv_dest.gid_present = source->gid_present =
01666                                 MLX_GET ( &cqe->normal, g );
01667                         memcpy ( &recv_dest.gid, &grh->dgid,
01668                                  sizeof ( recv_dest.gid ) );
01669                         memcpy ( &source->gid, &grh->sgid,
01670                                  sizeof ( source->gid ) );
01671                         break;
01672                 case IB_QPT_RC:
01673                         source = &qp->av;
01674                         break;
01675                 default:
01676                         assert ( 0 );
01677                         return -EINVAL;
01678                 }
01679                 assert ( len <= iob_tailroom ( iobuf ) );
01680                 iob_put ( iobuf, len );
01681                 /* Hand off to completion handler */
01682                 ib_complete_recv ( ibdev, qp, &recv_dest, source, iobuf, rc );
01683         }
01684 
01685         return rc;
01686 }                            
01687 
01688 /**
01689  * Poll completion queue
01690  *
01691  * @v ibdev             Infiniband device
01692  * @v cq                Completion queue
01693  */
01694 static void arbel_poll_cq ( struct ib_device *ibdev,
01695                             struct ib_completion_queue *cq ) {
01696         struct arbel *arbel = ib_get_drvdata ( ibdev );
01697         struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
01698         struct arbelprm_cq_ci_db_record *ci_db_rec;
01699         union arbelprm_completion_entry *cqe;
01700         unsigned int cqe_idx_mask;
01701         int rc;
01702 
01703         while ( 1 ) {
01704                 /* Look for completion entry */
01705                 cqe_idx_mask = ( cq->num_cqes - 1 );
01706                 cqe = &arbel_cq->cqe[cq->next_idx & cqe_idx_mask];
01707                 if ( MLX_GET ( &cqe->normal, owner ) != 0 ) {
01708                         /* Entry still owned by hardware; end of poll */
01709                         break;
01710                 }
01711 
01712                 /* Handle completion */
01713                 if ( ( rc = arbel_complete ( ibdev, cq, cqe ) ) != 0 ) {
01714                         DBGC ( arbel, "Arbel %p CQN %#lx failed to complete: "
01715                                "%s\n", arbel, cq->cqn, strerror ( rc ) );
01716                         DBGC_HD ( arbel, cqe, sizeof ( *cqe ) );
01717                 }
01718 
01719                 /* Return ownership to hardware */
01720                 MLX_FILL_1 ( &cqe->normal, 7, owner, 1 );
01721                 barrier();
01722                 /* Update completion queue's index */
01723                 cq->next_idx++;
01724                 /* Update doorbell record */
01725                 ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
01726                 MLX_FILL_1 ( ci_db_rec, 0,
01727                              counter, ( cq->next_idx & 0xffffffffUL ) );
01728         }
01729 }
01730 
01731 /***************************************************************************
01732  *
01733  * Event queues
01734  *
01735  ***************************************************************************
01736  */
01737 
01738 /**
01739  * Create event queue
01740  *
01741  * @v arbel             Arbel device
01742  * @ret rc              Return status code
01743  */
01744 static int arbel_create_eq ( struct arbel *arbel ) {
01745         struct arbel_event_queue *arbel_eq = &arbel->eq;
01746         struct arbelprm_eqc eqctx;
01747         struct arbelprm_event_mask mask;
01748         unsigned int i;
01749         int rc;
01750 
01751         /* Select event queue number */
01752         arbel_eq->eqn = arbel->limits.reserved_eqs;
01753 
01754         /* Calculate doorbell address */
01755         arbel_eq->doorbell = ( arbel->eq_ci_doorbells +
01756                                ARBEL_DB_EQ_OFFSET ( arbel_eq->eqn ) );
01757 
01758         /* Allocate event queue itself */
01759         arbel_eq->eqe_size =
01760                 ( ARBEL_NUM_EQES * sizeof ( arbel_eq->eqe[0] ) );
01761         arbel_eq->eqe = malloc_dma ( arbel_eq->eqe_size,
01762                                      sizeof ( arbel_eq->eqe[0] ) );
01763         if ( ! arbel_eq->eqe ) {
01764                 rc = -ENOMEM;
01765                 goto err_eqe;
01766         }
01767         memset ( arbel_eq->eqe, 0, arbel_eq->eqe_size );
01768         for ( i = 0 ; i < ARBEL_NUM_EQES ; i++ ) {
01769                 MLX_FILL_1 ( &arbel_eq->eqe[i].generic, 7, owner, 1 );
01770         }
01771         barrier();
01772 
01773         /* Hand queue over to hardware */
01774         memset ( &eqctx, 0, sizeof ( eqctx ) );
01775         MLX_FILL_1 ( &eqctx, 0, st, 0xa /* "Fired" */ );
01776         MLX_FILL_H ( &eqctx, 1,
01777                      start_address_h, virt_to_phys ( arbel_eq->eqe ) );
01778         MLX_FILL_1 ( &eqctx, 2,
01779                      start_address_l, virt_to_phys ( arbel_eq->eqe ) );
01780         MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( ARBEL_NUM_EQES - 1 ) );
01781         MLX_FILL_1 ( &eqctx, 6, pd, ARBEL_GLOBAL_PD );
01782         MLX_FILL_1 ( &eqctx, 7, lkey, arbel->lkey );
01783         if ( ( rc = arbel_cmd_sw2hw_eq ( arbel, arbel_eq->eqn,
01784                                          &eqctx ) ) != 0 ) {
01785                 DBGC ( arbel, "Arbel %p EQN %#lx SW2HW_EQ failed: %s\n",
01786                        arbel, arbel_eq->eqn, strerror ( rc ) );
01787                 goto err_sw2hw_eq;
01788         }
01789 
01790         /* Map events to this event queue */
01791         memset ( &mask, 0xff, sizeof ( mask ) );
01792         if ( ( rc = arbel_cmd_map_eq ( arbel,
01793                                        ( ARBEL_MAP_EQ | arbel_eq->eqn ),
01794                                        &mask ) ) != 0 ) {
01795                 DBGC ( arbel, "Arbel %p EQN %#lx MAP_EQ failed: %s\n",
01796                        arbel, arbel_eq->eqn, strerror ( rc )  );
01797                 goto err_map_eq;
01798         }
01799 
01800         DBGC ( arbel, "Arbel %p EQN %#lx ring [%08lx,%08lx), doorbell %08lx\n",
01801                arbel, arbel_eq->eqn, virt_to_phys ( arbel_eq->eqe ),
01802                ( virt_to_phys ( arbel_eq->eqe ) + arbel_eq->eqe_size ),
01803                virt_to_phys ( arbel_eq->doorbell ) );
01804         return 0;
01805 
01806  err_map_eq:
01807         arbel_cmd_hw2sw_eq ( arbel, arbel_eq->eqn, &eqctx );
01808  err_sw2hw_eq:
01809         free_dma ( arbel_eq->eqe, arbel_eq->eqe_size );
01810  err_eqe:
01811         memset ( arbel_eq, 0, sizeof ( *arbel_eq ) );
01812         return rc;
01813 }
01814 
01815 /**
01816  * Destroy event queue
01817  *
01818  * @v arbel             Arbel device
01819  */
01820 static void arbel_destroy_eq ( struct arbel *arbel ) {
01821         struct arbel_event_queue *arbel_eq = &arbel->eq;
01822         struct arbelprm_eqc eqctx;
01823         struct arbelprm_event_mask mask;
01824         int rc;
01825 
01826         /* Unmap events from event queue */
01827         memset ( &mask, 0, sizeof ( mask ) );
01828         MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
01829         if ( ( rc = arbel_cmd_map_eq ( arbel,
01830                                        ( ARBEL_UNMAP_EQ | arbel_eq->eqn ),
01831                                        &mask ) ) != 0 ) {
01832                 DBGC ( arbel, "Arbel %p EQN %#lx FATAL MAP_EQ failed to "
01833                        "unmap: %s\n", arbel, arbel_eq->eqn, strerror ( rc ) );
01834                 /* Continue; HCA may die but system should survive */
01835         }
01836 
01837         /* Take ownership back from hardware */
01838         if ( ( rc = arbel_cmd_hw2sw_eq ( arbel, arbel_eq->eqn,
01839                                          &eqctx ) ) != 0 ) {
01840                 DBGC ( arbel, "Arbel %p EQN %#lx FATAL HW2SW_EQ failed: %s\n",
01841                        arbel, arbel_eq->eqn, strerror ( rc ) );
01842                 /* Leak memory and return; at least we avoid corruption */
01843                 return;
01844         }
01845 
01846         /* Free memory */
01847         free_dma ( arbel_eq->eqe, arbel_eq->eqe_size );
01848         memset ( arbel_eq, 0, sizeof ( *arbel_eq ) );
01849 }
01850 
01851 /**
01852  * Handle port state event
01853  *
01854  * @v arbel             Arbel device
01855  * @v eqe               Port state change event queue entry
01856  */
01857 static void arbel_event_port_state_change ( struct arbel *arbel,
01858                                             union arbelprm_event_entry *eqe){
01859         unsigned int port;
01860         int link_up;
01861 
01862         /* Get port and link status */
01863         port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
01864         link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
01865         DBGC ( arbel, "Arbel %p port %d link %s\n", arbel, ( port + 1 ),
01866                ( link_up ? "up" : "down" ) );
01867 
01868         /* Sanity check */
01869         if ( port >= ARBEL_NUM_PORTS ) {
01870                 DBGC ( arbel, "Arbel %p port %d does not exist!\n",
01871                        arbel, ( port + 1 ) );
01872                 return;
01873         }
01874 
01875         /* Update MAD parameters */
01876         ib_smc_update ( arbel->ibdev[port], arbel_mad );
01877 }
01878 
01879 /**
01880  * Poll event queue
01881  *
01882  * @v ibdev             Infiniband device
01883  */
01884 static void arbel_poll_eq ( struct ib_device *ibdev ) {
01885         struct arbel *arbel = ib_get_drvdata ( ibdev );
01886         struct arbel_event_queue *arbel_eq = &arbel->eq;
01887         union arbelprm_event_entry *eqe;
01888         union arbelprm_eq_doorbell_register db_reg;
01889         unsigned int eqe_idx_mask;
01890         unsigned int event_type;
01891 
01892         /* No event is generated upon reaching INIT, so we must poll
01893          * separately for link state changes while we remain DOWN.
01894          */
01895         if ( ib_is_open ( ibdev ) &&
01896              ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) {
01897                 ib_smc_update ( ibdev, arbel_mad );
01898         }
01899 
01900         /* Poll event queue */
01901         while ( 1 ) {
01902                 /* Look for event entry */
01903                 eqe_idx_mask = ( ARBEL_NUM_EQES - 1 );
01904                 eqe = &arbel_eq->eqe[arbel_eq->next_idx & eqe_idx_mask];
01905                 if ( MLX_GET ( &eqe->generic, owner ) != 0 ) {
01906                         /* Entry still owned by hardware; end of poll */
01907                         break;
01908                 }
01909                 DBGCP ( arbel, "Arbel %p EQN %#lx event:\n",
01910                         arbel, arbel_eq->eqn );
01911                 DBGCP_HDA ( arbel, virt_to_phys ( eqe ),
01912                             eqe, sizeof ( *eqe ) );
01913 
01914                 /* Handle event */
01915                 event_type = MLX_GET ( &eqe->generic, event_type );
01916                 switch ( event_type ) {
01917                 case ARBEL_EV_PORT_STATE_CHANGE:
01918                         arbel_event_port_state_change ( arbel, eqe );
01919                         break;
01920                 default:
01921                         DBGC ( arbel, "Arbel %p EQN %#lx unrecognised event "
01922                                "type %#x:\n",
01923                                arbel, arbel_eq->eqn, event_type );
01924                         DBGC_HDA ( arbel, virt_to_phys ( eqe ),
01925                                    eqe, sizeof ( *eqe ) );
01926                         break;
01927                 }
01928 
01929                 /* Return ownership to hardware */
01930                 MLX_FILL_1 ( &eqe->generic, 7, owner, 1 );
01931                 barrier();
01932 
01933                 /* Update event queue's index */
01934                 arbel_eq->next_idx++;
01935 
01936                 /* Ring doorbell */
01937                 MLX_FILL_1 ( &db_reg.ci, 0, ci, arbel_eq->next_idx );
01938                 writel ( db_reg.dword[0], arbel_eq->doorbell );
01939         }
01940 }
01941 
01942 /***************************************************************************
01943  *
01944  * Firmware control
01945  *
01946  ***************************************************************************
01947  */
01948 
01949 /**
01950  * Map virtual to physical address for firmware usage
01951  *
01952  * @v arbel             Arbel device
01953  * @v map               Mapping function
01954  * @v va                Virtual address
01955  * @v pa                Physical address
01956  * @v len               Length of region
01957  * @ret rc              Return status code
01958  */
01959 static int arbel_map_vpm ( struct arbel *arbel,
01960                            int ( *map ) ( struct arbel *arbel,
01961                              const struct arbelprm_virtual_physical_mapping* ),
01962                            uint64_t va, physaddr_t pa, size_t len ) {
01963         struct arbelprm_virtual_physical_mapping mapping;
01964         physaddr_t start;
01965         physaddr_t low;
01966         physaddr_t high;
01967         physaddr_t end;
01968         size_t size;
01969         int rc;
01970 
01971         /* Sanity checks */
01972         assert ( ( va & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
01973         assert ( ( pa & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
01974         assert ( ( len & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
01975         assert ( len != 0 );
01976 
01977         /* Calculate starting points */
01978         start = pa;
01979         end = ( start + len );
01980         size = ( 1UL << ( fls ( start ^ end ) - 1 ) );
01981         low = high = ( end & ~( size - 1 ) );
01982         assert ( start < low );
01983         assert ( high <= end );
01984 
01985         /* These mappings tend to generate huge volumes of
01986          * uninteresting debug data, which basically makes it
01987          * impossible to use debugging otherwise.
01988          */
01989         DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
01990 
01991         /* Map blocks in descending order of size */
01992         while ( size >= ARBEL_PAGE_SIZE ) {
01993 
01994                 /* Find the next candidate block */
01995                 if ( ( low - size ) >= start ) {
01996                         low -= size;
01997                         pa = low;
01998                 } else if ( high <= ( end - size ) ) {
01999                         pa = high;
02000                         high += size;
02001                 } else {
02002                         size >>= 1;
02003                         continue;
02004                 }
02005                 assert ( ( va & ( size - 1 ) ) == 0 );
02006                 assert ( ( pa & ( size - 1 ) ) == 0 );
02007 
02008                 /* Map this block */
02009                 memset ( &mapping, 0, sizeof ( mapping ) );
02010                 MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) );
02011                 MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) );
02012                 MLX_FILL_H ( &mapping, 2, pa_h, pa );
02013                 MLX_FILL_2 ( &mapping, 3,
02014                              log2size, ( ( fls ( size ) - 1 ) - 12 ),
02015                              pa_l, ( pa >> 12 ) );
02016                 if ( ( rc = map ( arbel, &mapping ) ) != 0 ) {
02017                         DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
02018                         DBGC ( arbel, "Arbel %p could not map %08llx+%zx to "
02019                                "%08lx: %s\n",
02020                                arbel, va, size, pa, strerror ( rc ) );
02021                         return rc;
02022                 }
02023                 va += size;
02024         }
02025         assert ( low == start );
02026         assert ( high == end );
02027 
02028         DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
02029         return 0;
02030 }
02031 
02032 /**
02033  * Start firmware running
02034  *
02035  * @v arbel             Arbel device
02036  * @ret rc              Return status code
02037  */
02038 static int arbel_start_firmware ( struct arbel *arbel ) {
02039         struct arbelprm_query_fw fw;
02040         struct arbelprm_access_lam lam;
02041         unsigned int fw_pages;
02042         size_t fw_len;
02043         physaddr_t fw_base;
02044         uint64_t eq_set_ci_base_addr;
02045         int rc;
02046 
02047         /* Get firmware parameters */
02048         if ( ( rc = arbel_cmd_query_fw ( arbel, &fw ) ) != 0 ) {
02049                 DBGC ( arbel, "Arbel %p could not query firmware: %s\n",
02050                        arbel, strerror ( rc ) );
02051                 goto err_query_fw;
02052         }
02053         DBGC ( arbel, "Arbel %p firmware version %d.%d.%d\n", arbel,
02054                MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
02055                MLX_GET ( &fw, fw_rev_subminor ) );
02056         fw_pages = MLX_GET ( &fw, fw_pages );
02057         DBGC ( arbel, "Arbel %p requires %d kB for firmware\n",
02058                arbel, ( fw_pages * 4 ) );
02059         eq_set_ci_base_addr =
02060                 ( ( (uint64_t) MLX_GET ( &fw, eq_set_ci_base_addr_h ) << 32 ) |
02061                   ( (uint64_t) MLX_GET ( &fw, eq_set_ci_base_addr_l ) ) );
02062         arbel->eq_ci_doorbells = ioremap ( eq_set_ci_base_addr, 0x200 );
02063 
02064         /* Enable locally-attached memory.  Ignore failure; there may
02065          * be no attached memory.
02066          */
02067         arbel_cmd_enable_lam ( arbel, &lam );
02068 
02069         /* Allocate firmware pages and map firmware area */
02070         fw_len = ( fw_pages * ARBEL_PAGE_SIZE );
02071         if ( ! arbel->firmware_area ) {
02072                 arbel->firmware_len = fw_len;
02073                 arbel->firmware_area = umalloc ( arbel->firmware_len );
02074                 if ( ! arbel->firmware_area ) {
02075                         rc = -ENOMEM;
02076                         goto err_alloc_fa;
02077                 }
02078         } else {
02079                 assert ( arbel->firmware_len == fw_len );
02080         }
02081         fw_base = user_to_phys ( arbel->firmware_area, 0 );
02082         DBGC ( arbel, "Arbel %p firmware area at [%08lx,%08lx)\n",
02083                arbel, fw_base, ( fw_base + fw_len ) );
02084         if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_fa,
02085                                     0, fw_base, fw_len ) ) != 0 ) {
02086                 DBGC ( arbel, "Arbel %p could not map firmware: %s\n",
02087                        arbel, strerror ( rc ) );
02088                 goto err_map_fa;
02089         }
02090 
02091         /* Start firmware */
02092         if ( ( rc = arbel_cmd_run_fw ( arbel ) ) != 0 ) {
02093                 DBGC ( arbel, "Arbel %p could not run firmware: %s\n",
02094                        arbel, strerror ( rc ) );
02095                 goto err_run_fw;
02096         }
02097 
02098         DBGC ( arbel, "Arbel %p firmware started\n", arbel );
02099         return 0;
02100 
02101  err_run_fw:
02102         arbel_cmd_unmap_fa ( arbel );
02103  err_map_fa:
02104  err_alloc_fa:
02105  err_query_fw:
02106         return rc;
02107 }
02108 
02109 /**
02110  * Stop firmware running
02111  *
02112  * @v arbel             Arbel device
02113  */
02114 static void arbel_stop_firmware ( struct arbel *arbel ) {
02115         int rc;
02116 
02117         if ( ( rc = arbel_cmd_unmap_fa ( arbel ) ) != 0 ) {
02118                 DBGC ( arbel, "Arbel %p FATAL could not stop firmware: %s\n",
02119                        arbel, strerror ( rc ) );
02120                 /* Leak memory and return; at least we avoid corruption */
02121                 arbel->firmware_area = UNULL;
02122                 return;
02123         }
02124 }
02125 
02126 /***************************************************************************
02127  *
02128  * Infinihost Context Memory management
02129  *
02130  ***************************************************************************
02131  */
02132 
02133 /**
02134  * Get device limits
02135  *
02136  * @v arbel             Arbel device
02137  * @ret rc              Return status code
02138  */
02139 static int arbel_get_limits ( struct arbel *arbel ) {
02140         struct arbelprm_query_dev_lim dev_lim;
02141         int rc;
02142 
02143         if ( ( rc = arbel_cmd_query_dev_lim ( arbel, &dev_lim ) ) != 0 ) {
02144                 DBGC ( arbel, "Arbel %p could not get device limits: %s\n",
02145                        arbel, strerror ( rc ) );
02146                 return rc;
02147         }
02148 
02149         arbel->limits.reserved_qps =
02150                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_qps ) );
02151         arbel->limits.qpc_entry_size = MLX_GET ( &dev_lim, qpc_entry_sz );
02152         arbel->limits.eqpc_entry_size = MLX_GET ( &dev_lim, eqpc_entry_sz );
02153         arbel->limits.reserved_srqs =
02154                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_srqs ) );
02155         arbel->limits.srqc_entry_size = MLX_GET ( &dev_lim, srq_entry_sz );
02156         arbel->limits.reserved_ees =
02157                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_ees ) );
02158         arbel->limits.eec_entry_size = MLX_GET ( &dev_lim, eec_entry_sz );
02159         arbel->limits.eeec_entry_size = MLX_GET ( &dev_lim, eeec_entry_sz );
02160         arbel->limits.reserved_cqs =
02161                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_cqs ) );
02162         arbel->limits.cqc_entry_size = MLX_GET ( &dev_lim, cqc_entry_sz );
02163         arbel->limits.reserved_mtts =
02164                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_mtts ) );
02165         arbel->limits.mtt_entry_size = MLX_GET ( &dev_lim, mtt_entry_sz );
02166         arbel->limits.reserved_mrws =
02167                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_mrws ) );
02168         arbel->limits.mpt_entry_size = MLX_GET ( &dev_lim, mpt_entry_sz );
02169         arbel->limits.reserved_rdbs =
02170                 ( 1 << MLX_GET ( &dev_lim, log2_rsvd_rdbs ) );
02171         arbel->limits.reserved_eqs = MLX_GET ( &dev_lim, num_rsvd_eqs );
02172         arbel->limits.eqc_entry_size = MLX_GET ( &dev_lim, eqc_entry_sz );
02173         arbel->limits.reserved_uars = MLX_GET ( &dev_lim, num_rsvd_uars );
02174         arbel->limits.uar_scratch_entry_size =
02175                 MLX_GET ( &dev_lim, uar_scratch_entry_sz );
02176 
02177         DBGC ( arbel, "Arbel %p reserves %d x %#zx QPC, %d x %#zx EQPC, "
02178                "%d x %#zx SRQC\n", arbel,
02179                arbel->limits.reserved_qps, arbel->limits.qpc_entry_size,
02180                arbel->limits.reserved_qps, arbel->limits.eqpc_entry_size,
02181                arbel->limits.reserved_srqs, arbel->limits.srqc_entry_size );
02182         DBGC ( arbel, "Arbel %p reserves %d x %#zx EEC, %d x %#zx EEEC, "
02183                "%d x %#zx CQC\n", arbel,
02184                arbel->limits.reserved_ees, arbel->limits.eec_entry_size,
02185                arbel->limits.reserved_ees, arbel->limits.eeec_entry_size,
02186                arbel->limits.reserved_cqs, arbel->limits.cqc_entry_size );
02187         DBGC ( arbel, "Arbel %p reserves %d x %#zx EQC, %d x %#zx MTT, "
02188                "%d x %#zx MPT\n", arbel,
02189                arbel->limits.reserved_eqs, arbel->limits.eqc_entry_size,
02190                arbel->limits.reserved_mtts, arbel->limits.mtt_entry_size,
02191                arbel->limits.reserved_mrws, arbel->limits.mpt_entry_size );
02192         DBGC ( arbel, "Arbel %p reserves %d x %#zx RDB, %d x %#zx UAR, "
02193                "%d x %#zx UAR scratchpad\n", arbel,
02194                arbel->limits.reserved_rdbs, ARBEL_RDB_ENTRY_SIZE,
02195                arbel->limits.reserved_uars, ARBEL_PAGE_SIZE,
02196                arbel->limits.reserved_uars,
02197                arbel->limits.uar_scratch_entry_size );
02198 
02199         return 0;
02200 }
02201 
02202 /**
02203  * Align ICM table
02204  *
02205  * @v icm_offset        Current ICM offset
02206  * @v len               ICM table length
02207  * @ret icm_offset      ICM offset
02208  */
02209 static size_t icm_align ( size_t icm_offset, size_t len ) {
02210 
02211         /* Round up to a multiple of the table size */
02212         assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
02213         return ( ( icm_offset + len - 1 ) & ~( len - 1 ) );
02214 }
02215 
02216 /**
02217  * Allocate ICM
02218  *
02219  * @v arbel             Arbel device
02220  * @v init_hca          INIT_HCA structure to fill in
02221  * @ret rc              Return status code
02222  */
02223 static int arbel_alloc_icm ( struct arbel *arbel,
02224                              struct arbelprm_init_hca *init_hca ) {
02225         struct arbelprm_scalar_parameter icm_size;
02226         struct arbelprm_scalar_parameter icm_aux_size;
02227         struct arbelprm_scalar_parameter unmap_icm;
02228         union arbelprm_doorbell_record *db_rec;
02229         size_t icm_offset = 0;
02230         unsigned int log_num_uars, log_num_qps, log_num_srqs, log_num_ees;
02231         unsigned int log_num_cqs, log_num_mtts, log_num_mpts, log_num_rdbs;
02232         unsigned int log_num_eqs, log_num_mcs;
02233         size_t icm_len, icm_aux_len;
02234         size_t len;
02235         physaddr_t icm_phys;
02236         int rc;
02237 
02238         /* Calculate number of each object type within ICM */
02239         log_num_qps = fls ( arbel->limits.reserved_qps +
02240                             ARBEL_RSVD_SPECIAL_QPS + ARBEL_MAX_QPS - 1 );
02241         log_num_srqs = fls ( arbel->limits.reserved_srqs - 1 );
02242         log_num_ees = fls ( arbel->limits.reserved_ees - 1 );
02243         log_num_cqs = fls ( arbel->limits.reserved_cqs + ARBEL_MAX_CQS - 1 );
02244         log_num_eqs = fls ( arbel->limits.reserved_eqs + ARBEL_MAX_EQS - 1 );
02245         log_num_mtts = fls ( arbel->limits.reserved_mtts - 1 );
02246         log_num_mpts = fls ( arbel->limits.reserved_mrws + 1 - 1 );
02247         log_num_rdbs = fls ( arbel->limits.reserved_rdbs +
02248                              ARBEL_RSVD_SPECIAL_QPS + ARBEL_MAX_QPS - 1 );
02249         log_num_uars = fls ( arbel->limits.reserved_uars +
02250                              1 /* single UAR used */ - 1 );
02251         log_num_mcs = ARBEL_LOG_MULTICAST_HASH_SIZE;
02252 
02253         /* Queue pair contexts */
02254         len = ( ( 1 << log_num_qps ) * arbel->limits.qpc_entry_size );
02255         icm_offset = icm_align ( icm_offset, len );
02256         MLX_FILL_2 ( init_hca, 13,
02257                      qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l,
02258                      ( icm_offset >> 7 ),
02259                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
02260                      log_num_qps );
02261         DBGC ( arbel, "Arbel %p ICM QPC is %d x %#zx at [%zx,%zx)\n",
02262                arbel, ( 1 << log_num_qps ), arbel->limits.qpc_entry_size,
02263                icm_offset, ( icm_offset + len ) );
02264         icm_offset += len;
02265 
02266         /* Extended queue pair contexts */
02267         len = ( ( 1 << log_num_qps ) * arbel->limits.eqpc_entry_size );
02268         icm_offset = icm_align ( icm_offset, len );
02269         MLX_FILL_1 ( init_hca, 25,
02270                      qpc_eec_cqc_eqc_rdb_parameters.eqpc_base_addr_l,
02271                      icm_offset );
02272         DBGC ( arbel, "Arbel %p ICM EQPC is %d x %#zx at [%zx,%zx)\n",
02273                arbel, ( 1 << log_num_qps ), arbel->limits.eqpc_entry_size,
02274                icm_offset, ( icm_offset + len ) );
02275         icm_offset += len;
02276 
02277         /* Completion queue contexts */
02278         len = ( ( 1 << log_num_cqs ) * arbel->limits.cqc_entry_size );
02279         icm_offset = icm_align ( icm_offset, len );
02280         MLX_FILL_2 ( init_hca, 21,
02281                      qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l,
02282                      ( icm_offset >> 6 ),
02283                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
02284                      log_num_cqs );
02285         DBGC ( arbel, "Arbel %p ICM CQC is %d x %#zx at [%zx,%zx)\n",
02286                arbel, ( 1 << log_num_cqs ), arbel->limits.cqc_entry_size,
02287                icm_offset, ( icm_offset + len ) );
02288         icm_offset += len;
02289 
02290         /* Event queue contexts */
02291         len = ( ( 1 << log_num_eqs ) * arbel->limits.eqc_entry_size );
02292         icm_offset = icm_align ( icm_offset, len );
02293         MLX_FILL_2 ( init_hca, 33,
02294                      qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l,
02295                      ( icm_offset >> 6 ),
02296                      qpc_eec_cqc_eqc_rdb_parameters.log_num_eq,
02297                      log_num_eqs );
02298         DBGC ( arbel, "Arbel %p ICM EQC is %d x %#zx at [%zx,%zx)\n",
02299                arbel, ( 1 << log_num_eqs ), arbel->limits.eqc_entry_size,
02300                icm_offset, ( icm_offset + len ) );
02301         icm_offset += len;
02302 
02303         /* End-to-end contexts */
02304         len = ( ( 1 << log_num_ees ) * arbel->limits.eec_entry_size );
02305         icm_offset = icm_align ( icm_offset, len );
02306         MLX_FILL_2 ( init_hca, 17,
02307                      qpc_eec_cqc_eqc_rdb_parameters.eec_base_addr_l,
02308                      ( icm_offset >> 7 ),
02309                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_ee,
02310                      log_num_ees );
02311         DBGC ( arbel, "Arbel %p ICM EEC is %d x %#zx at [%zx,%zx)\n",
02312                arbel, ( 1 << log_num_ees ), arbel->limits.eec_entry_size,
02313                icm_offset, ( icm_offset + len ) );
02314         icm_offset += len;
02315 
02316         /* Shared receive queue contexts */
02317         len = ( ( 1 << log_num_srqs ) * arbel->limits.srqc_entry_size );
02318         icm_offset = icm_align ( icm_offset, len );
02319         MLX_FILL_2 ( init_hca, 19,
02320                      qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l,
02321                      ( icm_offset >> 5 ),
02322                      qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
02323                      log_num_srqs );
02324         DBGC ( arbel, "Arbel %p ICM SRQC is %d x %#zx at [%zx,%zx)\n",
02325                arbel, ( 1 << log_num_srqs ), arbel->limits.srqc_entry_size,
02326                icm_offset, ( icm_offset + len ) );
02327         icm_offset += len;
02328 
02329         /* Memory protection table */
02330         len = ( ( 1 << log_num_mpts ) * arbel->limits.mpt_entry_size );
02331         icm_offset = icm_align ( icm_offset, len );
02332         MLX_FILL_1 ( init_hca, 61,
02333                      tpt_parameters.mpt_base_adr_l, icm_offset );
02334         MLX_FILL_1 ( init_hca, 62,
02335                      tpt_parameters.log_mpt_sz, log_num_mpts );
02336         DBGC ( arbel, "Arbel %p ICM MPT is %d x %#zx at [%zx,%zx)\n",
02337                arbel, ( 1 << log_num_mpts ), arbel->limits.mpt_entry_size,
02338                icm_offset, ( icm_offset + len ) );
02339         icm_offset += len;
02340 
02341         /* Remote read data base table */
02342         len = ( ( 1 << log_num_rdbs ) * ARBEL_RDB_ENTRY_SIZE );
02343         icm_offset = icm_align ( icm_offset, len );
02344         MLX_FILL_1 ( init_hca, 37,
02345                      qpc_eec_cqc_eqc_rdb_parameters.rdb_base_addr_l,
02346                      icm_offset );
02347         DBGC ( arbel, "Arbel %p ICM RDB is %d x %#zx at [%zx,%zx)\n",
02348                arbel, ( 1 << log_num_rdbs ), ARBEL_RDB_ENTRY_SIZE,
02349                icm_offset, ( icm_offset + len ) );
02350         icm_offset += len;
02351 
02352         /* Extended end-to-end contexts */
02353         len = ( ( 1 << log_num_ees ) * arbel->limits.eeec_entry_size );
02354         icm_offset = icm_align ( icm_offset, len );
02355         MLX_FILL_1 ( init_hca, 29,
02356                      qpc_eec_cqc_eqc_rdb_parameters.eeec_base_addr_l,
02357                      icm_offset );
02358         DBGC ( arbel, "Arbel %p ICM EEEC is %d x %#zx at [%zx,%zx)\n",
02359                arbel, ( 1 << log_num_ees ), arbel->limits.eeec_entry_size,
02360                icm_offset, ( icm_offset + len ) );
02361         icm_offset += len;
02362 
02363         /* Multicast table */
02364         len = ( ( 1 << log_num_mcs ) * sizeof ( struct arbelprm_mgm_entry ) );
02365         icm_offset = icm_align ( icm_offset, len );
02366         MLX_FILL_1 ( init_hca, 49,
02367                      multicast_parameters.mc_base_addr_l, icm_offset );
02368         MLX_FILL_1 ( init_hca, 52,
02369                      multicast_parameters.log_mc_table_entry_sz,
02370                      fls ( sizeof ( struct arbelprm_mgm_entry ) - 1 ) );
02371         MLX_FILL_1 ( init_hca, 53,
02372                      multicast_parameters.mc_table_hash_sz,
02373                      ( 1 << log_num_mcs ) );
02374         MLX_FILL_1 ( init_hca, 54,
02375                      multicast_parameters.log_mc_table_sz,
02376                      log_num_mcs /* Only one entry per hash */ );
02377         DBGC ( arbel, "Arbel %p ICM MC is %d x %#zx at [%zx,%zx)\n", arbel,
02378                ( 1 << log_num_mcs ), sizeof ( struct arbelprm_mgm_entry ),
02379                icm_offset, ( icm_offset + len ) );
02380         icm_offset += len;
02381 
02382         /* Memory translation table */
02383         len = ( ( 1 << log_num_mtts ) * arbel->limits.mtt_entry_size );
02384         icm_offset = icm_align ( icm_offset, len );
02385         MLX_FILL_1 ( init_hca, 65,
02386                      tpt_parameters.mtt_base_addr_l, icm_offset );
02387         DBGC ( arbel, "Arbel %p ICM MTT is %d x %#zx at [%zx,%zx)\n",
02388                arbel, ( 1 << log_num_mtts ), arbel->limits.mtt_entry_size,
02389                icm_offset, ( icm_offset + len ) );
02390         icm_offset += len;
02391 
02392         /* User access region scratchpads */
02393         len = ( ( 1 << log_num_uars ) * arbel->limits.uar_scratch_entry_size );
02394         icm_offset = icm_align ( icm_offset, len );
02395         MLX_FILL_1 ( init_hca, 77,
02396                      uar_parameters.uar_scratch_base_addr_l, icm_offset );
02397         DBGC ( arbel, "Arbel %p UAR scratchpad is %d x %#zx at [%zx,%zx)\n",
02398                arbel, ( 1 << log_num_uars ),
02399                arbel->limits.uar_scratch_entry_size,
02400                icm_offset, ( icm_offset + len ) );
02401         icm_offset += len;
02402 
02403         /* Record amount of ICM to be allocated */
02404         icm_offset = icm_align ( icm_offset, ARBEL_PAGE_SIZE );
02405         icm_len = icm_offset;
02406 
02407         /* User access region contexts
02408          *
02409          * The reserved UAR(s) do not need to be backed by physical
02410          * memory, and our UAR is allocated separately; neither are
02411          * part of the umalloc()ed ICM block, but both contribute to
02412          * the total length of ICM virtual address space.
02413          */
02414         len = ( ( 1 << log_num_uars ) * ARBEL_PAGE_SIZE );
02415         icm_offset = icm_align ( icm_offset, len );
02416         MLX_FILL_1 ( init_hca, 74, uar_parameters.log_max_uars, log_num_uars );
02417         MLX_FILL_1 ( init_hca, 79,
02418                      uar_parameters.uar_context_base_addr_l, icm_offset );
02419         arbel->db_rec_offset =
02420                 ( icm_offset +
02421                   ( arbel->limits.reserved_uars * ARBEL_PAGE_SIZE ) );
02422         DBGC ( arbel, "Arbel %p UAR is %d x %#zx at [%zx,%zx), doorbells "
02423                "[%zx,%zx)\n", arbel, ( 1 << log_num_uars ), ARBEL_PAGE_SIZE,
02424                icm_offset, ( icm_offset + len ), arbel->db_rec_offset,
02425                ( arbel->db_rec_offset + ARBEL_PAGE_SIZE ) );
02426         icm_offset += len;
02427 
02428         /* Get ICM auxiliary area size */
02429         memset ( &icm_size, 0, sizeof ( icm_size ) );
02430         MLX_FILL_1 ( &icm_size, 1, value, icm_len );
02431         if ( ( rc = arbel_cmd_set_icm_size ( arbel, &icm_size,
02432                                              &icm_aux_size ) ) != 0 ) {
02433                 DBGC ( arbel, "Arbel %p could not set ICM size: %s\n",
02434                        arbel, strerror ( rc ) );
02435                 goto err_set_icm_size;
02436         }
02437         icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * ARBEL_PAGE_SIZE );
02438 
02439         /* Allocate ICM data and auxiliary area */
02440         DBGC ( arbel, "Arbel %p requires %zd kB ICM and %zd kB AUX ICM\n",
02441                arbel, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
02442         if ( ! arbel->icm ) {
02443                 arbel->icm_len = icm_len;
02444                 arbel->icm_aux_len = icm_aux_len;
02445                 arbel->icm = umalloc ( arbel->icm_len + arbel->icm_aux_len );
02446                 if ( ! arbel->icm ) {
02447                         rc = -ENOMEM;
02448                         goto err_alloc_icm;
02449                 }
02450         } else {
02451                 assert ( arbel->icm_len == icm_len );
02452                 assert ( arbel->icm_aux_len == icm_aux_len );
02453         }
02454         icm_phys = user_to_phys ( arbel->icm, 0 );
02455 
02456         /* Allocate doorbell UAR */
02457         arbel->db_rec = malloc_dma ( ARBEL_PAGE_SIZE, ARBEL_PAGE_SIZE );
02458         if ( ! arbel->db_rec ) {
02459                 rc = -ENOMEM;
02460                 goto err_alloc_doorbell;
02461         }
02462 
02463         /* Map ICM auxiliary area */
02464         DBGC ( arbel, "Arbel %p ICM AUX at [%08lx,%08lx)\n",
02465                arbel, icm_phys, ( icm_phys + arbel->icm_aux_len ) );
02466         if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_icm_aux,
02467                                     0, icm_phys, arbel->icm_aux_len ) ) != 0 ){
02468                 DBGC ( arbel, "Arbel %p could not map AUX ICM: %s\n",
02469                        arbel, strerror ( rc ) );
02470                 goto err_map_icm_aux;
02471         }
02472         icm_phys += arbel->icm_aux_len;
02473 
02474         /* Map ICM area */
02475         DBGC ( arbel, "Arbel %p ICM at [%08lx,%08lx)\n",
02476                arbel, icm_phys, ( icm_phys + arbel->icm_len ) );
02477         if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_icm,
02478                                     0, icm_phys, arbel->icm_len ) ) != 0 ) {
02479                 DBGC ( arbel, "Arbel %p could not map ICM: %s\n",
02480                        arbel, strerror ( rc ) );
02481                 goto err_map_icm;
02482         }
02483         icm_phys += arbel->icm_len;
02484 
02485         /* Map doorbell UAR */
02486         DBGC ( arbel, "Arbel %p UAR at [%08lx,%08lx)\n",
02487                arbel, virt_to_phys ( arbel->db_rec ),
02488                ( virt_to_phys ( arbel->db_rec ) + ARBEL_PAGE_SIZE ) );
02489         if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_icm,
02490                                     arbel->db_rec_offset,
02491                                     virt_to_phys ( arbel->db_rec ),
02492                                     ARBEL_PAGE_SIZE ) ) != 0 ) {
02493                 DBGC ( arbel, "Arbel %p could not map doorbell UAR: %s\n",
02494                        arbel, strerror ( rc ) );
02495                 goto err_map_doorbell;
02496         }
02497 
02498         /* Initialise doorbell records */
02499         memset ( arbel->db_rec, 0, ARBEL_PAGE_SIZE );
02500         db_rec = &arbel->db_rec[ARBEL_GROUP_SEPARATOR_DOORBELL];
02501         MLX_FILL_1 ( &db_rec->qp, 1, res, ARBEL_UAR_RES_GROUP_SEP );
02502 
02503         return 0;
02504 
02505         memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
02506         MLX_FILL_1 ( &unmap_icm, 1, value, arbel->db_rec_offset );
02507         arbel_cmd_unmap_icm ( arbel, 1, &unmap_icm );
02508  err_map_doorbell:
02509         memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
02510         arbel_cmd_unmap_icm ( arbel, ( arbel->icm_len / ARBEL_PAGE_SIZE ),
02511                               &unmap_icm );
02512  err_map_icm:
02513         arbel_cmd_unmap_icm_aux ( arbel );
02514  err_map_icm_aux:
02515         free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
02516         arbel->db_rec= NULL;
02517  err_alloc_doorbell:
02518  err_alloc_icm:
02519  err_set_icm_size:
02520         return rc;
02521 }
02522 
02523 /**
02524  * Free ICM
02525  *
02526  * @v arbel             Arbel device
02527  */
02528 static void arbel_free_icm ( struct arbel *arbel ) {
02529         struct arbelprm_scalar_parameter unmap_icm;
02530 
02531         memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
02532         MLX_FILL_1 ( &unmap_icm, 1, value, arbel->db_rec_offset );
02533         arbel_cmd_unmap_icm ( arbel, 1, &unmap_icm );
02534         memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
02535         arbel_cmd_unmap_icm ( arbel, ( arbel->icm_len / ARBEL_PAGE_SIZE ),
02536                               &unmap_icm );
02537         arbel_cmd_unmap_icm_aux ( arbel );
02538         free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
02539         arbel->db_rec = NULL;
02540 }
02541 
02542 /***************************************************************************
02543  *
02544  * Initialisation and teardown
02545  *
02546  ***************************************************************************
02547  */
02548 
02549 /**
02550  * Reset device
02551  *
02552  * @v arbel             Arbel device
02553  */
02554 static void arbel_reset ( struct arbel *arbel ) {
02555         struct pci_device *pci = arbel->pci;
02556         struct pci_config_backup backup;
02557         static const uint8_t backup_exclude[] =
02558                 PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
02559         uint16_t vendor;
02560         unsigned int i;
02561 
02562         /* Perform device reset and preserve PCI configuration */
02563         pci_backup ( pci, &backup, backup_exclude );
02564         writel ( ARBEL_RESET_MAGIC,
02565                  ( arbel->config + ARBEL_RESET_OFFSET ) );
02566         for ( i = 0 ; i < ARBEL_RESET_WAIT_TIME_MS ; i++ ) {
02567                 mdelay ( 1 );
02568                 pci_read_config_word ( pci, PCI_VENDOR_ID, &vendor );
02569                 if ( vendor != 0xffff )
02570                         break;
02571         }
02572         pci_restore ( pci, &backup, backup_exclude );
02573 }
02574 
02575 /**
02576  * Set up memory protection table
02577  *
02578  * @v arbel             Arbel device
02579  * @ret rc              Return status code
02580  */
02581 static int arbel_setup_mpt ( struct arbel *arbel ) {
02582         struct arbelprm_mpt mpt;
02583         uint32_t key;
02584         int rc;
02585 
02586         /* Derive key */
02587         key = ( arbel->limits.reserved_mrws | ARBEL_MKEY_PREFIX );
02588         arbel->lkey = ( ( key << 8 ) | ( key >> 24 ) );
02589 
02590         /* Initialise memory protection table */
02591         memset ( &mpt, 0, sizeof ( mpt ) );
02592         MLX_FILL_7 ( &mpt, 0,
02593                      a, 1,
02594                      rw, 1,
02595                      rr, 1,
02596                      lw, 1,
02597                      lr, 1,
02598                      pa, 1,
02599                      r_w, 1 );
02600         MLX_FILL_1 ( &mpt, 2, mem_key, key );
02601         MLX_FILL_2 ( &mpt, 3,
02602                      pd, ARBEL_GLOBAL_PD,
02603                      rae, 1 );
02604         MLX_FILL_1 ( &mpt, 6, reg_wnd_len_h, 0xffffffffUL );
02605         MLX_FILL_1 ( &mpt, 7, reg_wnd_len_l, 0xffffffffUL );
02606         if ( ( rc = arbel_cmd_sw2hw_mpt ( arbel, arbel->limits.reserved_mrws,
02607                                           &mpt ) ) != 0 ) {
02608                 DBGC ( arbel, "Arbel %p could not set up MPT: %s\n",
02609                        arbel, strerror ( rc ) );
02610                 return rc;
02611         }
02612 
02613         return 0;
02614 }
02615 
02616 /**
02617  * Configure special queue pairs
02618  *
02619  * @v arbel             Arbel device
02620  * @ret rc              Return status code
02621  */
02622 static int arbel_configure_special_qps ( struct arbel *arbel ) {
02623         unsigned int smi_qpn_base;
02624         unsigned int gsi_qpn_base;
02625         int rc;
02626 
02627         /* Special QP block must be aligned on an even number */
02628         arbel->special_qpn_base = ( ( arbel->limits.reserved_qps + 1 ) & ~1 );
02629         arbel->qpn_base = ( arbel->special_qpn_base +
02630                             ARBEL_NUM_SPECIAL_QPS );
02631         DBGC ( arbel, "Arbel %p special QPs at [%lx,%lx]\n", arbel,
02632                arbel->special_qpn_base, ( arbel->qpn_base - 1 ) );
02633         smi_qpn_base = arbel->special_qpn_base;
02634         gsi_qpn_base = ( smi_qpn_base + 2 );
02635 
02636         /* Issue commands to configure special QPs */
02637         if ( ( rc = arbel_cmd_conf_special_qp ( arbel, 0,
02638                                                 smi_qpn_base ) ) != 0 ) {
02639                 DBGC ( arbel, "Arbel %p could not configure SMI QPs: %s\n",
02640                        arbel, strerror ( rc ) );
02641                 return rc;
02642         }
02643         if ( ( rc = arbel_cmd_conf_special_qp ( arbel, 1,
02644                                                 gsi_qpn_base ) ) != 0 ) {
02645                 DBGC ( arbel, "Arbel %p could not configure GSI QPs: %s\n",
02646                        arbel, strerror ( rc ) );
02647                 return rc;
02648         }
02649 
02650         return 0;
02651 }
02652 
02653 /**
02654  * Start Arbel device
02655  *
02656  * @v arbel             Arbel device
02657  * @v running           Firmware is already running
02658  * @ret rc              Return status code
02659  */
02660 static int arbel_start ( struct arbel *arbel, int running ) {
02661         struct arbelprm_init_hca init_hca;
02662         unsigned int i;
02663         int rc;
02664 
02665         /* Start firmware if not already running */
02666         if ( ! running ) {
02667                 if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
02668                         goto err_start_firmware;
02669         }
02670 
02671         /* Allocate ICM */
02672         memset ( &init_hca, 0, sizeof ( init_hca ) );
02673         if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
02674                 goto err_alloc_icm;
02675 
02676         /* Initialise HCA */
02677         if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
02678                 DBGC ( arbel, "Arbel %p could not initialise HCA: %s\n",
02679                        arbel, strerror ( rc ) );
02680                 goto err_init_hca;
02681         }
02682 
02683         /* Set up memory protection */
02684         if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
02685                 goto err_setup_mpt;
02686         for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
02687                 arbel->ibdev[i]->rdma_key = arbel->lkey;
02688 
02689         /* Set up event queue */
02690         if ( ( rc = arbel_create_eq ( arbel ) ) != 0 )
02691                 goto err_create_eq;
02692 
02693         /* Configure special QPs */
02694         if ( ( rc = arbel_configure_special_qps ( arbel ) ) != 0 )
02695                 goto err_conf_special_qps;
02696 
02697         return 0;
02698 
02699  err_conf_special_qps:
02700         arbel_destroy_eq ( arbel );
02701  err_create_eq:
02702  err_setup_mpt:
02703         arbel_cmd_close_hca ( arbel );
02704  err_init_hca:
02705         arbel_free_icm ( arbel );
02706  err_alloc_icm:
02707         arbel_stop_firmware ( arbel );
02708  err_start_firmware:
02709         return rc;
02710 }
02711 
02712 /**
02713  * Stop Arbel device
02714  *
02715  * @v arbel             Arbel device
02716  */
02717 static void arbel_stop ( struct arbel *arbel ) {
02718         arbel_destroy_eq ( arbel );
02719         arbel_cmd_close_hca ( arbel );
02720         arbel_free_icm ( arbel );
02721         arbel_stop_firmware ( arbel );
02722         arbel_reset ( arbel );
02723 }
02724 
02725 /**
02726  * Open Arbel device
02727  *
02728  * @v arbel             Arbel device
02729  * @ret rc              Return status code
02730  */
02731 static int arbel_open ( struct arbel *arbel ) {
02732         int rc;
02733 
02734         /* Start device if applicable */
02735         if ( arbel->open_count == 0 ) {
02736                 if ( ( rc = arbel_start ( arbel, 0 ) ) != 0 )
02737                         return rc;
02738         }
02739 
02740         /* Increment open counter */
02741         arbel->open_count++;
02742 
02743         return 0;
02744 }
02745 
02746 /**
02747  * Close Arbel device
02748  *
02749  * @v arbel             Arbel device
02750  */
02751 static void arbel_close ( struct arbel *arbel ) {
02752 
02753         /* Decrement open counter */
02754         assert ( arbel->open_count != 0 );
02755         arbel->open_count--;
02756 
02757         /* Stop device if applicable */
02758         if ( arbel->open_count == 0 )
02759                 arbel_stop ( arbel );
02760 }
02761 
02762 /***************************************************************************
02763  *
02764  * Infiniband link-layer operations
02765  *
02766  ***************************************************************************
02767  */
02768 
02769 /**
02770  * Initialise Infiniband link
02771  *
02772  * @v ibdev             Infiniband device
02773  * @ret rc              Return status code
02774  */
02775 static int arbel_ib_open ( struct ib_device *ibdev ) {
02776         struct arbel *arbel = ib_get_drvdata ( ibdev );
02777         struct arbelprm_init_ib init_ib;
02778         int rc;
02779 
02780         /* Open hardware */
02781         if ( ( rc = arbel_open ( arbel ) ) != 0 )
02782                 goto err_open;
02783 
02784         /* Initialise IB */
02785         memset ( &init_ib, 0, sizeof ( init_ib ) );
02786         MLX_FILL_3 ( &init_ib, 0,
02787                      mtu_cap, ARBEL_MTU_2048,
02788                      port_width_cap, 3,
02789                      vl_cap, 1 );
02790         MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
02791         MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
02792         if ( ( rc = arbel_cmd_init_ib ( arbel, ibdev->port,
02793                                         &init_ib ) ) != 0 ) {
02794                 DBGC ( arbel, "Arbel %p port %d could not intialise IB: %s\n",
02795                        arbel, ibdev->port, strerror ( rc ) );
02796                 goto err_init_ib;
02797         }
02798 
02799         /* Update MAD parameters */
02800         ib_smc_update ( ibdev, arbel_mad );
02801 
02802         return 0;
02803 
02804  err_init_ib:
02805         arbel_close ( arbel );
02806  err_open:
02807         return rc;
02808 }
02809 
02810 /**
02811  * Close Infiniband link
02812  *
02813  * @v ibdev             Infiniband device
02814  */
02815 static void arbel_ib_close ( struct ib_device *ibdev ) {
02816         struct arbel *arbel = ib_get_drvdata ( ibdev );
02817         int rc;
02818 
02819         /* Close IB */
02820         if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
02821                 DBGC ( arbel, "Arbel %p port %d could not close IB: %s\n",
02822                        arbel, ibdev->port, strerror ( rc ) );
02823                 /* Nothing we can do about this */
02824         }
02825 
02826         /* Close hardware */
02827         arbel_close ( arbel );
02828 }
02829 
02830 /**
02831  * Inform embedded subnet management agent of a received MAD
02832  *
02833  * @v ibdev             Infiniband device
02834  * @v mad               MAD
02835  * @ret rc              Return status code
02836  */
02837 static int arbel_inform_sma ( struct ib_device *ibdev, union ib_mad *mad ) {
02838         int rc;
02839 
02840         /* Send the MAD to the embedded SMA */
02841         if ( ( rc = arbel_mad ( ibdev, mad ) ) != 0 )
02842                 return rc;
02843 
02844         /* Update parameters held in software */
02845         ib_smc_update ( ibdev, arbel_mad );
02846 
02847         return 0;
02848 }
02849 
02850 /***************************************************************************
02851  *
02852  * Multicast group operations
02853  *
02854  ***************************************************************************
02855  */
02856 
02857 /**
02858  * Attach to multicast group
02859  *
02860  * @v ibdev             Infiniband device
02861  * @v qp                Queue pair
02862  * @v gid               Multicast GID
02863  * @ret rc              Return status code
02864  */
02865 static int arbel_mcast_attach ( struct ib_device *ibdev,
02866                                 struct ib_queue_pair *qp,
02867                                 union ib_gid *gid ) {
02868         struct arbel *arbel = ib_get_drvdata ( ibdev );
02869         struct arbelprm_mgm_hash hash;
02870         struct arbelprm_mgm_entry mgm;
02871         unsigned int index;
02872         int rc;
02873 
02874         /* Generate hash table index */
02875         if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) {
02876                 DBGC ( arbel, "Arbel %p could not hash GID: %s\n",
02877                        arbel, strerror ( rc ) );
02878                 return rc;
02879         }
02880         index = MLX_GET ( &hash, hash );
02881 
02882         /* Check for existing hash table entry */
02883         if ( ( rc = arbel_cmd_read_mgm ( arbel, index, &mgm ) ) != 0 ) {
02884                 DBGC ( arbel, "Arbel %p could not read MGM %#x: %s\n",
02885                        arbel, index, strerror ( rc ) );
02886                 return rc;
02887         }
02888         if ( MLX_GET ( &mgm, mgmqp_0.qi ) != 0 ) {
02889                 /* FIXME: this implementation allows only a single QP
02890                  * per multicast group, and doesn't handle hash
02891                  * collisions.  Sufficient for IPoIB but may need to
02892                  * be extended in future.
02893                  */
02894                 DBGC ( arbel, "Arbel %p MGID index %#x already in use\n",
02895                        arbel, index );
02896                 return -EBUSY;
02897         }
02898 
02899         /* Update hash table entry */
02900         MLX_FILL_2 ( &mgm, 8,
02901                      mgmqp_0.qpn_i, qp->qpn,
02902                      mgmqp_0.qi, 1 );
02903         memcpy ( &mgm.u.dwords[4], gid, sizeof ( *gid ) );
02904         if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) {
02905                 DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n",
02906                        arbel, index, strerror ( rc ) );
02907                 return rc;
02908         }
02909 
02910         return 0;
02911 }
02912 
02913 /**
02914  * Detach from multicast group
02915  *
02916  * @v ibdev             Infiniband device
02917  * @v qp                Queue pair
02918  * @v gid               Multicast GID
02919  */
02920 static void arbel_mcast_detach ( struct ib_device *ibdev,
02921                                  struct ib_queue_pair *qp __unused,
02922                                  union ib_gid *gid ) {
02923         struct arbel *arbel = ib_get_drvdata ( ibdev );
02924         struct arbelprm_mgm_hash hash;
02925         struct arbelprm_mgm_entry mgm;
02926         unsigned int index;
02927         int rc;
02928 
02929         /* Generate hash table index */
02930         if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) {
02931                 DBGC ( arbel, "Arbel %p could not hash GID: %s\n",
02932                        arbel, strerror ( rc ) );
02933                 return;
02934         }
02935         index = MLX_GET ( &hash, hash );
02936 
02937         /* Clear hash table entry */
02938         memset ( &mgm, 0, sizeof ( mgm ) );
02939         if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) {
02940                 DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n",
02941                        arbel, index, strerror ( rc ) );
02942                 return;
02943         }
02944 }
02945 
02946 /** Arbel Infiniband operations */
02947 static struct ib_device_operations arbel_ib_operations = {
02948         .create_cq      = arbel_create_cq,
02949         .destroy_cq     = arbel_destroy_cq,
02950         .create_qp      = arbel_create_qp,
02951         .modify_qp      = arbel_modify_qp,
02952         .destroy_qp     = arbel_destroy_qp,
02953         .post_send      = arbel_post_send,
02954         .post_recv      = arbel_post_recv,
02955         .poll_cq        = arbel_poll_cq,
02956         .poll_eq        = arbel_poll_eq,
02957         .open           = arbel_ib_open,
02958         .close          = arbel_ib_close,
02959         .mcast_attach   = arbel_mcast_attach,
02960         .mcast_detach   = arbel_mcast_detach,
02961         .set_port_info  = arbel_inform_sma,
02962         .set_pkey_table = arbel_inform_sma,
02963 };
02964 
02965 /***************************************************************************
02966  *
02967  * PCI interface
02968  *
02969  ***************************************************************************
02970  */
02971 
02972 /**
02973  * Allocate Arbel device
02974  *
02975  * @ret arbel           Arbel device
02976  */
02977 static struct arbel * arbel_alloc ( void ) {
02978         struct arbel *arbel;
02979 
02980         /* Allocate Arbel device */
02981         arbel = zalloc ( sizeof ( *arbel ) );
02982         if ( ! arbel )
02983                 goto err_arbel;
02984 
02985         /* Allocate space for mailboxes */
02986         arbel->mailbox_in = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
02987         if ( ! arbel->mailbox_in )
02988                 goto err_mailbox_in;
02989         arbel->mailbox_out = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
02990         if ( ! arbel->mailbox_out )
02991                 goto err_mailbox_out;
02992 
02993         return arbel;
02994 
02995         free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
02996  err_mailbox_out:
02997         free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
02998  err_mailbox_in:
02999         free ( arbel );
03000  err_arbel:
03001         return NULL;
03002 }
03003 
03004 /**
03005  * Free Arbel device
03006  *
03007  * @v arbel             Arbel device
03008  */
03009 static void arbel_free ( struct arbel *arbel ) {
03010 
03011         ufree ( arbel->icm );
03012         ufree ( arbel->firmware_area );
03013         free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
03014         free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
03015         free ( arbel );
03016 }
03017 
03018 /**
03019  * Probe PCI device
03020  *
03021  * @v pci               PCI device
03022  * @v id                PCI ID
03023  * @ret rc              Return status code
03024  */
03025 static int arbel_probe ( struct pci_device *pci ) {
03026         struct arbel *arbel;
03027         struct ib_device *ibdev;
03028         int i;
03029         int rc;
03030 
03031         /* Allocate Arbel device */
03032         arbel = arbel_alloc();
03033         if ( ! arbel ) {
03034                 rc = -ENOMEM;
03035                 goto err_alloc;
03036         }
03037         pci_set_drvdata ( pci, arbel );
03038         arbel->pci = pci;
03039 
03040         /* Fix up PCI device */
03041         adjust_pci_device ( pci );
03042 
03043         /* Map PCI BARs */
03044         arbel->config = ioremap ( pci_bar_start ( pci, ARBEL_PCI_CONFIG_BAR ),
03045                                   ARBEL_PCI_CONFIG_BAR_SIZE );
03046         arbel->uar = ioremap ( ( pci_bar_start ( pci, ARBEL_PCI_UAR_BAR ) +
03047                                  ARBEL_PCI_UAR_IDX * ARBEL_PCI_UAR_SIZE ),
03048                                ARBEL_PCI_UAR_SIZE );
03049 
03050         /* Allocate Infiniband devices */
03051         for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
03052                 ibdev = alloc_ibdev ( 0 );
03053                 if ( ! ibdev ) {
03054                         rc = -ENOMEM;
03055                         goto err_alloc_ibdev;
03056                 }
03057                 arbel->ibdev[i] = ibdev;
03058                 ibdev->op = &arbel_ib_operations;
03059                 ibdev->dev = &pci->dev;
03060                 ibdev->port = ( ARBEL_PORT_BASE + i );
03061                 ib_set_drvdata ( ibdev, arbel );
03062         }
03063 
03064         /* Reset device */
03065         arbel_reset ( arbel );
03066 
03067         /* Start firmware */
03068         if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
03069                 goto err_start_firmware;
03070 
03071         /* Get device limits */
03072         if ( ( rc = arbel_get_limits ( arbel ) ) != 0 )
03073                 goto err_get_limits;
03074 
03075         /* Start device */
03076         if ( ( rc = arbel_start ( arbel, 1 ) ) != 0 )
03077                 goto err_start;
03078 
03079         /* Initialise parameters using SMC */
03080         for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
03081                 ib_smc_init ( arbel->ibdev[i], arbel_mad );
03082 
03083         /* Register Infiniband devices */
03084         for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
03085                 if ( ( rc = register_ibdev ( arbel->ibdev[i] ) ) != 0 ) {
03086                         DBGC ( arbel, "Arbel %p port %d could not register IB "
03087                                "device: %s\n", arbel,
03088                                arbel->ibdev[i]->port, strerror ( rc ) );
03089                         goto err_register_ibdev;
03090                 }
03091         }
03092 
03093         /* Leave device quiescent until opened */
03094         if ( arbel->open_count == 0 )
03095                 arbel_stop ( arbel );
03096 
03097         return 0;
03098 
03099         i = ARBEL_NUM_PORTS;
03100  err_register_ibdev:
03101         for ( i-- ; i >= 0 ; i-- )
03102                 unregister_ibdev ( arbel->ibdev[i] );
03103         arbel_stop ( arbel );
03104  err_start:
03105  err_get_limits:
03106         arbel_stop_firmware ( arbel );
03107  err_start_firmware:
03108         i = ARBEL_NUM_PORTS;
03109  err_alloc_ibdev:
03110         for ( i-- ; i >= 0 ; i-- )
03111                 ibdev_put ( arbel->ibdev[i] );
03112         iounmap ( arbel->uar );
03113         iounmap ( arbel->config );
03114         arbel_free ( arbel );
03115  err_alloc:
03116         return rc;
03117 }
03118 
03119 /**
03120  * Remove PCI device
03121  *
03122  * @v pci               PCI device
03123  */
03124 static void arbel_remove ( struct pci_device *pci ) {
03125         struct arbel *arbel = pci_get_drvdata ( pci );
03126         int i;
03127 
03128         for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
03129                 unregister_ibdev ( arbel->ibdev[i] );
03130         for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
03131                 ibdev_put ( arbel->ibdev[i] );
03132         iounmap ( arbel->uar );
03133         iounmap ( arbel->config );
03134         arbel_free ( arbel );
03135 }
03136 
03137 static struct pci_device_id arbel_nics[] = {
03138         PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
03139         PCI_ROM ( 0x15b3, 0x6274, "mt25204", "MT25204 HCA driver", 0 ),
03140 };
03141 
03142 struct pci_driver arbel_driver __pci_driver = {
03143         .ids = arbel_nics,
03144         .id_count = ( sizeof ( arbel_nics ) / sizeof ( arbel_nics[0] ) ),
03145         .probe = arbel_probe,
03146         .remove = arbel_remove,
03147 };