iPXE
ib_mcast.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <string.h>
28 #include <byteswap.h>
29 #include <errno.h>
30 #include <ipxe/list.h>
31 #include <ipxe/infiniband.h>
32 #include <ipxe/ib_mi.h>
33 #include <ipxe/ib_mcast.h>
34 
35 /** @file
36  *
37  * Infiniband multicast groups
38  *
39  */
40 
41 /**
42  * Generate multicast membership MAD
43  *
44  * @v ibdev Infiniband device
45  * @v av Address vector
46  * @v method Method (IB_MGMT_METHOD_SET or IB_MGMT_METHOD_DELETE)
47  * @v mask Additional component mask
48  * @v mad MAD to fill in
49  */
50 static void ib_mcast_mad ( struct ib_device *ibdev,
51  struct ib_address_vector *av,
52  unsigned int method, unsigned int mask,
53  union ib_mad *mad ) {
54  struct ib_mad_sa *sa = &mad->sa;
55 
56  /* Construct multicast membership record request */
57  memset ( sa, 0, sizeof ( *sa ) );
58  sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
59  sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
60  sa->mad_hdr.method = method;
61  sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
62  sa->sa_hdr.comp_mask[1] =
65  sa->sa_data.mc_member_record.qkey = htonl ( av->qkey );
66  sa->sa_data.mc_member_record.pkey =
67  htons ( ibdev->pkey | IB_PKEY_FULL );
68  sa->sa_data.mc_member_record.rate_selector__rate = av->rate;
69  sa->sa_data.mc_member_record.sl__flow_label__hop_limit =
70  htonl ( av->sl << 28 );
71  sa->sa_data.mc_member_record.scope__join_state = 0x01;
72  memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
73  sizeof ( sa->sa_data.mc_member_record.mgid ) );
74  memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
75  sizeof ( sa->sa_data.mc_member_record.port_gid ) );
76 }
77 
78 /**
79  * Handle multicast membership record join response
80  *
81  * @v ibdev Infiniband device
82  * @v mi Management interface
83  * @v madx Management transaction
84  * @v rc Status code
85  * @v mad Received MAD (or NULL on error)
86  * @v src Source address vector (or NULL on error)
87  */
88 static void ib_mcast_complete ( struct ib_device *ibdev,
89  struct ib_mad_interface *mi __unused,
90  struct ib_mad_transaction *madx,
91  int rc, union ib_mad *mad,
92  struct ib_address_vector *src __unused ) {
93  struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
94  struct ib_queue_pair *qp = membership->qp;
95  struct ib_address_vector *av = membership->av;
98  int joined;
99 
100  /* Report failures */
101  if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
102  rc = -ENOTCONN;
103  if ( rc != 0 ) {
104  DBGC ( ibdev, "IBDEV %s QPN %#lx join failed: %s\n",
105  ibdev->name, qp->qpn, strerror ( rc ) );
106  goto out;
107  }
108 
109  /* Extract values from MAD */
110  joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
111  av->qkey = ntohl ( mc_member_record->qkey );
112  av->lid = ntohs ( mc_member_record->mlid );
113  av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
115  >> 28 ) & 0x0f );
116  DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
117  ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
118  IB_GID_ARGS ( &av->gid ), av->qkey );
119 
120  /* Set queue key */
121  qp->qkey = av->qkey;
122  if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
123  DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
124  ibdev->name, qp->qpn, strerror ( rc ) );
125  goto out;
126  }
127 
128  out:
129  /* Destroy the completed transaction */
130  ib_destroy_madx ( ibdev, mi, madx );
131  membership->madx = NULL;
132 
133  /* Hand off to upper completion handler */
134  membership->complete ( membership, rc );
135 }
136 
137 /** Multicast membership management transaction completion operations */
140 };
141 
142 /**
143  * Join multicast group
144  *
145  * @v ibdev Infiniband device
146  * @v qp Queue pair
147  * @v membership Multicast group membership
148  * @v av Address vector to fill in
149  * @v joined Join completion handler
150  * @ret rc Return status code
151  */
152 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
153  struct ib_mc_membership *membership,
154  struct ib_address_vector *av, unsigned int mask,
155  void ( * complete ) ( struct ib_mc_membership *membership,
156  int rc ) ) {
157  union ib_mad mad;
158  int rc;
159 
160  DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
161  ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
162 
163  /* Sanity checks */
164  assert ( qp != NULL );
165  assert ( ! membership->attached );
166 
167  /* Initialise structure */
168  membership->qp = qp;
169  membership->av = av;
170  membership->complete = complete;
171 
172  /* Attach queue pair to multicast GID */
173  if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
174  DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
175  ibdev->name, qp->qpn, strerror ( rc ) );
176  goto err_mcast_attach;
177  }
178  membership->attached = 1;
179 
180  /* Initiate multicast membership join */
181  ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_SET, mask, &mad );
182  membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
183  &ib_mcast_op );
184  if ( ! membership->madx ) {
185  DBGC ( ibdev, "IBDEV %s QPN %#lx could not create join "
186  "transaction\n", ibdev->name, qp->qpn );
187  rc = -ENOMEM;
188  goto err_create_madx;
189  }
190  ib_madx_set_ownerdata ( membership->madx, membership );
191 
192  return 0;
193 
194  ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
195  err_create_madx:
196  ib_mcast_detach ( ibdev, qp, &av->gid );
197  membership->attached = 0;
198  err_mcast_attach:
199  return rc;
200 }
201 
202 /**
203  * Leave multicast group
204  *
205  * @v ibdev Infiniband device
206  * @v qp Queue pair
207  * @v membership Multicast group membership
208  */
209 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
210  struct ib_mc_membership *membership ) {
211  struct ib_address_vector *av = membership->av;
212  union ib_mad mad;
213  int rc;
214 
215  /* Do nothing if we are already detached from the multicast GID */
216  if ( ! membership->attached )
217  return;
218 
219  DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
220  ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
221 
222  /* Sanity check */
223  assert ( qp != NULL );
224 
225  /* Detach from multicast GID */
226  ib_mcast_detach ( ibdev, qp, &av->gid );
227  membership->attached = 0;
228 
229  /* Cancel multicast membership join, if applicable */
230  if ( membership->madx ) {
231  ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
232  membership->madx = NULL;
233  }
234 
235  /* Send a single group leave MAD */
236  ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_DELETE, 0, &mad );
237  if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
238  DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
239  "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );
240  }
241 }
#define IB_SA_CLASS_VERSION
Definition: ib_mad.h:207
uint8_t method
Definition: ib_mad.h:542
struct ib_mad_transaction * madx
Multicast group join transaction.
Definition: ib_mcast.h:25
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
Infiniband management interfaces.
#define IB_MGMT_METHOD_DELETE
Definition: ib_mad.h:577
Infiniband protocol.
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
#define IB_SA_ATTR_MC_MEMBER_REC
Definition: ib_mad.h:224
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
union ib_gid gid
Port GID (comprising GID prefix and port GUID)
Definition: infiniband.h:439
struct sockaddr sa
Definition: dns.c:68
Infiniband multicast groups.
static void ib_mcast_mad(struct ib_device *ibdev, struct ib_address_vector *av, unsigned int method, unsigned int mask, union ib_mad *mad)
Generate multicast membership MAD.
Definition: ib_mcast.c:50
Error codes.
int ib_mcast_join(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_mc_membership *membership, struct ib_address_vector *av, unsigned int mask, void(*complete)(struct ib_mc_membership *membership, int rc))
Join multicast group.
Definition: ib_mcast.c:152
#define IB_PKEY_FULL
Infiniband partition key full membership flag.
Definition: infiniband.h:42
int ib_modify_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Modify queue pair.
Definition: infiniband.c:294
#define DBGC(...)
Definition: compiler.h:505
struct ib_address_vector * av
Address vector.
Definition: ib_mcast.h:21
#define IB_SA_MCMEMBER_REC_MGID
Definition: ib_mad.h:278
int ib_mcast_attach(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Attach to multicast group.
Definition: infiniband.c:752
#define ntohl(value)
Definition: byteswap.h:134
struct ib_mad_interface * gsi
General services interface.
Definition: infiniband.h:459
#define ntohs(value)
Definition: byteswap.h:136
#define IB_MGMT_METHOD_SET
Definition: ib_mad.h:570
An Infiniband management interface.
Definition: ib_mi.h:88
uint8_t method
Definition: ib_mad.h:14
int attached
Attached to multicast GID.
Definition: ib_mcast.h:23
#define htonl(value)
Definition: byteswap.h:133
uint32_t sl__flow_label__hop_limit
Definition: ib_mad.h:271
enum ib_rate rate
Rate.
Definition: infiniband.h:86
An Infiniband device.
Definition: infiniband.h:398
#define ENOMEM
Not enough space.
Definition: errno.h:534
void * memcpy(void *dest, const void *src, size_t len) __nonnull
unsigned long qkey
Queue key.
Definition: infiniband.h:79
union ib_sa_data sa_data
Definition: ib_mad.h:600
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
An Infiniband management transaction.
Definition: ib_mi.h:70
static struct ib_mad_transaction_operations ib_mcast_op
Multicast membership management transaction completion operations.
Definition: ib_mcast.c:138
static void ib_mcast_complete(struct ib_device *ibdev, struct ib_mad_interface *mi __unused, struct ib_mad_transaction *madx, int rc, union ib_mad *mad, struct ib_address_vector *src __unused)
Handle multicast membership record join response.
Definition: ib_mcast.c:88
struct ib_mad_transaction * ib_create_madx(struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av, struct ib_mad_transaction_operations *op)
Create management transaction.
Definition: ib_mi.c:287
__be32 out[4]
Definition: CIB_PRM.h:36
static __always_inline void ib_madx_set_ownerdata(struct ib_mad_transaction *madx, void *priv)
Set Infiniband management transaction owner-private data.
Definition: ib_mi.h:106
An Infiniband multicast group membership.
Definition: ib_mcast.h:17
#define IB_MGMT_METHOD_GET_RESP
Definition: ib_mad.h:571
static __always_inline void off_t userptr_t src
Definition: efi_uaccess.h:66
#define IB_GID_ARGS(gid)
Infiniband Global Identifier debug message arguments.
Definition: ib_packet.h:48
#define ENOTCONN
The socket is not connected.
Definition: errno.h:569
Linked lists.
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
uint32_t qkey
Definition: ib_mad.h:264
void ib_mcast_detach(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Detach from multicast group.
Definition: infiniband.c:789
#define IB_SA_MCMEMBER_REC_JOIN_STATE
Definition: ib_mad.h:294
struct ib_mad_sa sa
Definition: ib_mad.h:613
static __always_inline void * ib_madx_get_ownerdata(struct ib_mad_transaction *madx)
Get Infiniband management transaction owner-private data.
Definition: ib_mi.h:117
struct ib_mc_member_record mc_member_record
Definition: ib_mad.h:300
An Infiniband Queue Pair.
Definition: infiniband.h:157
unsigned int sl
Service level.
Definition: infiniband.h:88
#define __unused
Declare a variable or data structure as unused.
Definition: compiler.h:573
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
#define IB_MGMT_CLASS_SUBN_ADM
Definition: ib_mad.h:557
struct ib_mad_hdr hdr
Definition: ib_mad.h:611
#define IB_MGMT_STATUS_OK
Definition: ib_mad.h:580
#define IB_SA_MCMEMBER_REC_PORT_GID
Definition: ib_mad.h:279
struct ib_queue_pair * qp
Queue pair.
Definition: ib_mcast.h:19
union ib_gid gid
GID, if present.
Definition: infiniband.h:92
void(* complete)(struct ib_mc_membership *membership, int rc)
Handle join success/failure.
Definition: ib_mcast.h:31
#define IB_GID_FMT
Infiniband Global Identifier debug message format.
Definition: ib_packet.h:45
A management datagram.
Definition: ib_mad.h:610
struct ib_mc_member_record mc_member_record
Definition: ib_mad.h:13
uint16_t pkey
Partition key.
Definition: infiniband.h:447
An Infiniband Address Vector.
Definition: infiniband.h:72
unsigned int lid
Local ID.
Definition: infiniband.h:81
void ib_mcast_leave(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_mc_membership *membership)
Leave multicast group.
Definition: ib_mcast.c:209
void(* complete)(struct ib_device *ibdev, struct ib_mad_interface *mi, struct ib_mad_transaction *madx, int rc, union ib_mad *mad, struct ib_address_vector *av)
Handle transaction completion.
Definition: ib_mi.h:62
Infiniband management transaction operations.
Definition: ib_mi.h:49
#define NULL
NULL pointer (VOID *)
Definition: Base.h:362
A subnet administration MAD.
Definition: ib_mad.h:596
uint8_t rate_selector__rate
Definition: ib_mad.h:269
String functions.
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77
#define htons(value)
Definition: byteswap.h:135
int ib_mi_send(struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
Transmit MAD.
Definition: ib_mi.c:187
uint16_t mlid
Definition: ib_mad.h:265
union ib_mad mad
Definition: arbel.h:12
uint16_t status
Definition: ib_mad.h:543
void ib_destroy_madx(struct ib_device *ibdev __unused, struct ib_mad_interface *mi __unused, struct ib_mad_transaction *madx)
Destroy management transaction.
Definition: ib_mi.c:327
void * memset(void *dest, int character, size_t len) __nonnull