iPXE
ib_mcast.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdint.h>
00027 #include <string.h>
00028 #include <byteswap.h>
00029 #include <errno.h>
00030 #include <ipxe/list.h>
00031 #include <ipxe/infiniband.h>
00032 #include <ipxe/ib_mi.h>
00033 #include <ipxe/ib_mcast.h>
00034 
00035 /** @file
00036  *
00037  * Infiniband multicast groups
00038  *
00039  */
00040 
00041 /**
00042  * Generate multicast membership MAD
00043  *
00044  * @v ibdev             Infiniband device
00045  * @v av                Address vector
00046  * @v method            Method (IB_MGMT_METHOD_SET or IB_MGMT_METHOD_DELETE)
00047  * @v mask              Additional component mask
00048  * @v mad               MAD to fill in
00049  */
00050 static void ib_mcast_mad ( struct ib_device *ibdev,
00051                            struct ib_address_vector *av,
00052                            unsigned int method, unsigned int mask,
00053                            union ib_mad *mad ) {
00054         struct ib_mad_sa *sa = &mad->sa;
00055 
00056         /* Construct multicast membership record request */
00057         memset ( sa, 0, sizeof ( *sa ) );
00058         sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
00059         sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
00060         sa->mad_hdr.method = method;
00061         sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
00062         sa->sa_hdr.comp_mask[1] =
00063                 htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
00064                         IB_SA_MCMEMBER_REC_JOIN_STATE | mask );
00065         sa->sa_data.mc_member_record.qkey = htonl ( av->qkey );
00066         sa->sa_data.mc_member_record.pkey =
00067                 htons ( ibdev->pkey | IB_PKEY_FULL );
00068         sa->sa_data.mc_member_record.rate_selector__rate = av->rate;
00069         sa->sa_data.mc_member_record.sl__flow_label__hop_limit =
00070                 htonl ( av->sl << 28 );
00071         sa->sa_data.mc_member_record.scope__join_state = 0x01;
00072         memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
00073                  sizeof ( sa->sa_data.mc_member_record.mgid ) );
00074         memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
00075                  sizeof ( sa->sa_data.mc_member_record.port_gid ) );
00076 }
00077 
00078 /**
00079  * Handle multicast membership record join response
00080  *
00081  * @v ibdev             Infiniband device
00082  * @v mi                Management interface
00083  * @v madx              Management transaction
00084  * @v rc                Status code
00085  * @v mad               Received MAD (or NULL on error)
00086  * @v src               Source address vector (or NULL on error)
00087  */
00088 static void ib_mcast_complete ( struct ib_device *ibdev,
00089                                 struct ib_mad_interface *mi __unused,
00090                                 struct ib_mad_transaction *madx,
00091                                 int rc, union ib_mad *mad,
00092                                 struct ib_address_vector *src __unused ) {
00093         struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
00094         struct ib_queue_pair *qp = membership->qp;
00095         struct ib_address_vector *av = membership->av;
00096         struct ib_mc_member_record *mc_member_record =
00097                 &mad->sa.sa_data.mc_member_record;
00098         int joined;
00099 
00100         /* Report failures */
00101         if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
00102                 rc = -ENOTCONN;
00103         if ( rc != 0 ) {
00104                 DBGC ( ibdev, "IBDEV %s QPN %#lx join failed: %s\n",
00105                        ibdev->name, qp->qpn, strerror ( rc ) );
00106                 goto out;
00107         }
00108 
00109         /* Extract values from MAD */
00110         joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
00111         av->qkey = ntohl ( mc_member_record->qkey );
00112         av->lid = ntohs ( mc_member_record->mlid );
00113         av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
00114         av->sl = ( ( ntohl ( mc_member_record->sl__flow_label__hop_limit )
00115                      >> 28 ) & 0x0f );
00116         DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
00117                ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
00118                IB_GID_ARGS ( &av->gid ), av->qkey );
00119 
00120         /* Set queue key */
00121         qp->qkey = av->qkey;
00122         if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
00123                 DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
00124                        ibdev->name, qp->qpn, strerror ( rc ) );
00125                 goto out;
00126         }
00127 
00128  out:
00129         /* Destroy the completed transaction */
00130         ib_destroy_madx ( ibdev, mi, madx );
00131         membership->madx = NULL;
00132 
00133         /* Hand off to upper completion handler */
00134         membership->complete ( membership, rc );
00135 }
00136 
00137 /** Multicast membership management transaction completion operations */
00138 static struct ib_mad_transaction_operations ib_mcast_op = {
00139         .complete = ib_mcast_complete,
00140 };
00141 
00142 /**
00143  * Join multicast group
00144  *
00145  * @v ibdev             Infiniband device
00146  * @v qp                Queue pair
00147  * @v membership        Multicast group membership
00148  * @v av                Address vector to fill in
00149  * @v joined            Join completion handler
00150  * @ret rc              Return status code
00151  */
00152 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
00153                     struct ib_mc_membership *membership,
00154                     struct ib_address_vector *av, unsigned int mask,
00155                     void ( * complete ) ( struct ib_mc_membership *membership,
00156                                           int rc ) ) {
00157         union ib_mad mad;
00158         int rc;
00159 
00160         DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
00161                ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
00162 
00163         /* Sanity checks */
00164         assert ( qp != NULL );
00165         assert ( ! membership->attached );
00166 
00167         /* Initialise structure */
00168         membership->qp = qp;
00169         membership->av = av;
00170         membership->complete = complete;
00171 
00172         /* Attach queue pair to multicast GID */
00173         if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
00174                 DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
00175                        ibdev->name, qp->qpn, strerror ( rc ) );
00176                 goto err_mcast_attach;
00177         }
00178         membership->attached = 1;
00179 
00180         /* Initiate multicast membership join */
00181         ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_SET, mask, &mad );
00182         membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
00183                                             &ib_mcast_op );
00184         if ( ! membership->madx ) {
00185                 DBGC ( ibdev, "IBDEV %s QPN %#lx could not create join "
00186                        "transaction\n", ibdev->name, qp->qpn );
00187                 rc = -ENOMEM;
00188                 goto err_create_madx;
00189         }
00190         ib_madx_set_ownerdata ( membership->madx, membership );
00191 
00192         return 0;
00193 
00194         ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
00195  err_create_madx:
00196         ib_mcast_detach ( ibdev, qp, &av->gid );
00197         membership->attached = 0;
00198  err_mcast_attach:
00199         return rc;
00200 }
00201 
00202 /**
00203  * Leave multicast group
00204  *
00205  * @v ibdev             Infiniband device
00206  * @v qp                Queue pair
00207  * @v membership        Multicast group membership
00208  */
00209 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
00210                       struct ib_mc_membership *membership ) {
00211         struct ib_address_vector *av = membership->av;
00212         union ib_mad mad;
00213         int rc;
00214 
00215         /* Do nothing if we are already detached from the multicast GID */
00216         if ( ! membership->attached )
00217                 return;
00218 
00219         DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
00220                ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
00221 
00222         /* Sanity check */
00223         assert ( qp != NULL );
00224 
00225         /* Detach from multicast GID */
00226         ib_mcast_detach ( ibdev, qp, &av->gid );
00227         membership->attached = 0;
00228 
00229         /* Cancel multicast membership join, if applicable */
00230         if ( membership->madx ) {
00231                 ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
00232                 membership->madx = NULL;
00233         }
00234 
00235         /* Send a single group leave MAD */
00236         ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_DELETE, 0, &mad );
00237         if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
00238                 DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
00239                        "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );
00240         }
00241 }