iPXE
ib_mi.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdint.h>
00027 #include <stdlib.h>
00028 #include <string.h>
00029 #include <errno.h>
00030 #include <stdio.h>
00031 #include <unistd.h>
00032 #include <byteswap.h>
00033 #include <ipxe/infiniband.h>
00034 #include <ipxe/iobuf.h>
00035 #include <ipxe/ib_mi.h>
00036 
00037 /**
00038  * @file
00039  *
00040  * Infiniband management interfaces
00041  *
00042  */
00043 
00044 /** Management interface number of send WQEs
00045  *
00046  * This is a policy decision.
00047  */
00048 #define IB_MI_NUM_SEND_WQES 4
00049 
00050 /** Management interface number of receive WQEs
00051  *
00052  * This is a policy decision.
00053  */
00054 #define IB_MI_NUM_RECV_WQES 2
00055 
00056 /** Management interface number of completion queue entries
00057  *
00058  * This is a policy decision
00059  */
00060 #define IB_MI_NUM_CQES 8
00061 
00062 /** TID magic signature */
00063 #define IB_MI_TID_MAGIC ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
00064 
00065 /** TID to use for next MAD */
00066 static unsigned int next_tid;
00067 
00068 /**
00069  * Handle received MAD
00070  *
00071  * @v ibdev             Infiniband device
00072  * @v mi                Management interface
00073  * @v mad               Received MAD
00074  * @v av                Source address vector
00075  * @ret rc              Return status code
00076  */
00077 static int ib_mi_handle ( struct ib_device *ibdev,
00078                           struct ib_mad_interface *mi,
00079                           union ib_mad *mad,
00080                           struct ib_address_vector *av ) {
00081         struct ib_mad_hdr *hdr = &mad->hdr;
00082         struct ib_mad_transaction *madx;
00083         struct ib_mad_agent *agent;
00084 
00085         /* Look for a matching transaction by TID */
00086         list_for_each_entry ( madx, &mi->madx, list ) {
00087                 if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
00088                               sizeof ( hdr->tid ) ) != 0 )
00089                         continue;
00090                 /* Found a matching transaction */
00091                 madx->op->complete ( ibdev, mi, madx, 0, mad, av );
00092                 return 0;
00093         }
00094 
00095         /* If there is no matching transaction, look for a listening agent */
00096         for_each_table_entry ( agent, IB_MAD_AGENTS ) {
00097                 if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
00098                        ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
00099                      ( agent->class_version != hdr->class_version ) ||
00100                      ( agent->attr_id != hdr->attr_id ) )
00101                         continue;
00102                 /* Found a matching agent */
00103                 agent->handle ( ibdev, mi, mad, av );
00104                 return 0;
00105         }
00106 
00107         /* Otherwise, ignore it */
00108         DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
00109                mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
00110         return -ENOTSUP;
00111 }
00112 
00113 /**
00114  * Complete receive via management interface
00115  *
00116  *
00117  * @v ibdev             Infiniband device
00118  * @v qp                Queue pair
00119  * @v dest              Destination address vector
00120  * @v source            Source address vector
00121  * @v iobuf             I/O buffer
00122  * @v rc                Completion status code
00123  */
00124 static void ib_mi_complete_recv ( struct ib_device *ibdev,
00125                                   struct ib_queue_pair *qp,
00126                                   struct ib_address_vector *dest __unused,
00127                                   struct ib_address_vector *source,
00128                                   struct io_buffer *iobuf, int rc ) {
00129         struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
00130         union ib_mad *mad;
00131         struct ib_mad_hdr *hdr;
00132 
00133         /* Ignore errors */
00134         if ( rc != 0 ) {
00135                 DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
00136                 goto out;
00137         }
00138 
00139         /* Sanity checks */
00140         if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
00141                 DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
00142                        mi, iob_len ( iobuf ) );
00143                 DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
00144                 goto out;
00145         }
00146         mad = iobuf->data;
00147         hdr = &mad->hdr;
00148         if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
00149                 DBGC ( mi, "MI %p RX unsupported base version %x\n",
00150                        mi, hdr->base_version );
00151                 DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
00152                 goto out;
00153         }
00154         DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
00155                "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
00156                hdr->mgmt_class, hdr->class_version, hdr->method,
00157                ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
00158         DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
00159 
00160         /* Handle MAD */
00161         if ( ( rc = ib_mi_handle ( ibdev, mi, mad, source ) ) != 0 )
00162                 goto out;
00163 
00164  out:
00165         free_iob ( iobuf );
00166 }
00167 
00168 /** Management interface completion operations */
00169 static struct ib_completion_queue_operations ib_mi_completion_ops = {
00170         .complete_recv = ib_mi_complete_recv,
00171 };
00172 
00173 /** Management interface queue pair operations */
00174 static struct ib_queue_pair_operations ib_mi_queue_pair_ops = {
00175         .alloc_iob = alloc_iob,
00176 };
00177 
00178 /**
00179  * Transmit MAD
00180  *
00181  * @v ibdev             Infiniband device
00182  * @v mi                Management interface
00183  * @v mad               MAD
00184  * @v av                Destination address vector
00185  * @ret rc              Return status code
00186  */
00187 int ib_mi_send ( struct ib_device *ibdev, struct ib_mad_interface *mi,
00188                  union ib_mad *mad, struct ib_address_vector *av ) {
00189         struct ib_mad_hdr *hdr = &mad->hdr;
00190         struct io_buffer *iobuf;
00191         int rc;
00192 
00193         /* Set common fields */
00194         hdr->base_version = IB_MGMT_BASE_VERSION;
00195         if ( ( hdr->tid.high == 0 ) && ( hdr->tid.low == 0 ) ) {
00196                 hdr->tid.high = htonl ( IB_MI_TID_MAGIC );
00197                 hdr->tid.low = htonl ( ++next_tid );
00198         }
00199         DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
00200                "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
00201                hdr->mgmt_class, hdr->class_version, hdr->method,
00202                ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
00203         DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
00204 
00205         /* Construct directed route portion of response, if necessary */
00206         if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
00207                 struct ib_mad_smp *smp = &mad->smp;
00208                 unsigned int hop_pointer;
00209                 unsigned int hop_count;
00210 
00211                 smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
00212                 hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
00213                 hop_count = smp->mad_hdr.class_specific.smp.hop_count;
00214                 assert ( hop_count == hop_pointer );
00215                 if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
00216                                      sizeof ( smp->return_path.hops[0] ) ) ) {
00217                         smp->return_path.hops[hop_pointer] = ibdev->port;
00218                 } else {
00219                         DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
00220                                "%d\n", mi, ntohl ( hdr->tid.high ),
00221                                ntohl ( hdr->tid.low ), hop_pointer );
00222                         return -EINVAL;
00223                 }
00224         }
00225 
00226         /* Construct I/O buffer */
00227         iobuf = alloc_iob ( sizeof ( *mad ) );
00228         if ( ! iobuf ) {
00229                 DBGC ( mi, "MI %p could not allocate buffer for TID "
00230                        "%08x%08x\n",
00231                        mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
00232                 return -ENOMEM;
00233         }
00234         memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );
00235 
00236         /* Send I/O buffer */
00237         if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
00238                 DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
00239                        mi,  ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
00240                        strerror ( rc ) );
00241                 free_iob ( iobuf );
00242                 return rc;
00243         }
00244 
00245         return 0;
00246 }
00247 
00248 /**
00249  * Handle management transaction timer expiry
00250  *
00251  * @v timer             Retry timer
00252  * @v expired           Failure indicator
00253  */
00254 static void ib_mi_timer_expired ( struct retry_timer *timer, int expired ) {
00255         struct ib_mad_transaction *madx =
00256                 container_of ( timer, struct ib_mad_transaction, timer );
00257         struct ib_mad_interface *mi = madx->mi;
00258         struct ib_device *ibdev = mi->ibdev;
00259         struct ib_mad_hdr *hdr = &madx->mad.hdr;
00260 
00261         /* Abandon transaction if we have tried too many times */
00262         if ( expired ) {
00263                 DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
00264                        mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
00265                 madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
00266                 return;
00267         }
00268 
00269         /* Restart retransmission timer */
00270         start_timer ( timer );
00271 
00272         /* Resend MAD */
00273         ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
00274 }
00275 
00276 /**
00277  * Create management transaction
00278  *
00279  * @v ibdev             Infiniband device
00280  * @v mi                Management interface
00281  * @v mad               MAD to send
00282  * @v av                Destination address, or NULL to use SM's GSI
00283  * @v op                Management transaction operations
00284  * @ret madx            Management transaction, or NULL
00285  */
00286 struct ib_mad_transaction *
00287 ib_create_madx ( struct ib_device *ibdev, struct ib_mad_interface *mi,
00288                  union ib_mad *mad, struct ib_address_vector *av,
00289                  struct ib_mad_transaction_operations *op ) {
00290         struct ib_mad_transaction *madx;
00291 
00292         /* Allocate and initialise structure */
00293         madx = zalloc ( sizeof ( *madx ) );
00294         if ( ! madx )
00295                 return NULL;
00296         timer_init ( &madx->timer, ib_mi_timer_expired, NULL );
00297         madx->mi = mi;
00298         madx->op = op;
00299 
00300         /* Determine address vector */
00301         if ( av ) {
00302                 memcpy ( &madx->av, av, sizeof ( madx->av ) );
00303         } else {
00304                 madx->av.lid = ibdev->sm_lid;
00305                 madx->av.sl = ibdev->sm_sl;
00306                 madx->av.qpn = IB_QPN_GSI;
00307                 madx->av.qkey = IB_QKEY_GSI;
00308         }
00309 
00310         /* Copy MAD */
00311         memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );
00312 
00313         /* Add to list and start timer to send initial MAD */
00314         list_add ( &madx->list, &mi->madx );
00315         start_timer_nodelay ( &madx->timer );
00316 
00317         return madx;
00318 }
00319 
00320 /**
00321  * Destroy management transaction
00322  *
00323  * @v ibdev             Infiniband device
00324  * @v mi                Management interface
00325  * @v madx              Management transaction
00326  */
00327 void ib_destroy_madx ( struct ib_device *ibdev __unused,
00328                        struct ib_mad_interface *mi __unused,
00329                        struct ib_mad_transaction *madx ) {
00330 
00331         /* Stop timer and remove from list */
00332         stop_timer ( &madx->timer );
00333         list_del ( &madx->list );
00334 
00335         /* Free transaction */
00336         free ( madx );
00337 }
00338 
00339 /**
00340  * Create management interface
00341  *
00342  * @v ibdev             Infiniband device
00343  * @v type              Queue pair type
00344  * @v new_mi            New management interface to fill in
00345  * @ret rc              Return status code
00346  */
00347 int ib_create_mi ( struct ib_device *ibdev, enum ib_queue_pair_type type,
00348                    struct ib_mad_interface **new_mi ) {
00349         struct ib_mad_interface *mi;
00350         const char *name;
00351         int rc;
00352 
00353         /* Allocate and initialise fields */
00354         mi = zalloc ( sizeof ( *mi ) );
00355         if ( ! mi ) {
00356                 rc = -ENOMEM;
00357                 goto err_alloc;
00358         }
00359         mi->ibdev = ibdev;
00360         INIT_LIST_HEAD ( &mi->madx );
00361 
00362         /* Create completion queue */
00363         if ( ( rc = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops,
00364                                    &mi->cq ) ) != 0 ) {
00365                 DBGC ( mi, "MI %p could not create completion queue: %s\n",
00366                        mi, strerror ( rc ) );
00367                 goto err_create_cq;
00368         }
00369 
00370         /* Create queue pair */
00371         name = ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" );
00372         if ( ( rc = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
00373                                    IB_MI_NUM_RECV_WQES, mi->cq,
00374                                    &ib_mi_queue_pair_ops, name, &mi->qp ) )!=0){
00375                 DBGC ( mi, "MI %p could not create queue pair: %s\n",
00376                        mi, strerror ( rc ) );
00377                 goto err_create_qp;
00378         }
00379         ib_qp_set_ownerdata ( mi->qp, mi );
00380         DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
00381                mi, mi->qp->name, mi->qp->qpn );
00382 
00383         /* Set queue key */
00384         mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
00385         if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
00386                 DBGC ( mi, "MI %p could not set queue key: %s\n",
00387                        mi, strerror ( rc ) );
00388                 goto err_modify_qp;
00389         }
00390 
00391         /* Fill receive ring */
00392         ib_refill_recv ( ibdev, mi->qp );
00393         *new_mi = mi;
00394         return 0;
00395 
00396  err_modify_qp:
00397         ib_destroy_qp ( ibdev, mi->qp );
00398  err_create_qp:
00399         ib_destroy_cq ( ibdev, mi->cq );
00400  err_create_cq:
00401         free ( mi );
00402  err_alloc:
00403         return rc;
00404 }
00405 
00406 /**
00407  * Destroy management interface
00408  *
00409  * @v mi                Management interface
00410  */
00411 void ib_destroy_mi ( struct ib_device *ibdev, struct ib_mad_interface *mi ) {
00412         struct ib_mad_transaction *madx;
00413         struct ib_mad_transaction *tmp;
00414 
00415         /* Flush any outstanding requests */
00416         list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
00417                 DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
00418                        mi, ntohl ( madx->mad.hdr.tid.high ),
00419                        ntohl ( madx->mad.hdr.tid.low ) );
00420                 madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
00421         }
00422 
00423         ib_destroy_qp ( ibdev, mi->qp );
00424         ib_destroy_cq ( ibdev, mi->cq );
00425         free ( mi );
00426 }