iPXE
Data Structures | Macros | Functions | Variables
ib_cmrc.c File Reference

Infiniband Communication-managed Reliable Connections. More...

#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <ipxe/iobuf.h>
#include <ipxe/xfer.h>
#include <ipxe/process.h>
#include <ipxe/infiniband.h>
#include <ipxe/ib_cm.h>
#include <ipxe/ib_cmrc.h>

Go to the source code of this file.

Data Structures

struct  ib_cmrc_connection
 An Infiniband Communication-Managed Reliable Connection. More...
 

Macros

#define IB_CMRC_NUM_SEND_WQES   4
 CMRC number of send WQEs. More...
 
#define IB_CMRC_NUM_RECV_WQES   2
 CMRC number of receive WQEs. More...
 
#define IB_CMRC_NUM_CQES   8
 CMRC number of completion queue entries. More...
 

Functions

 FILE_LICENCE (BSD2)
 
static void ib_cmrc_shutdown (struct ib_cmrc_connection *cmrc)
 Shut down CMRC connection gracefully. More...
 
static void ib_cmrc_close (struct ib_cmrc_connection *cmrc, int rc)
 Close CMRC connection. More...
 
static void ib_cmrc_changed (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_connection *conn __unused, int rc_cm, void *private_data, size_t private_data_len)
 Handle change of CMRC connection status. More...
 
static void ib_cmrc_complete_send (struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf, int rc)
 Handle CMRC send completion. More...
 
static void ib_cmrc_complete_recv (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest __unused, struct ib_address_vector *source __unused, struct io_buffer *iobuf, int rc)
 Handle CMRC receive completion. More...
 
static int ib_cmrc_xfer_deliver (struct ib_cmrc_connection *cmrc, struct io_buffer *iobuf, struct xfer_metadata *meta __unused)
 Send data via CMRC. More...
 
static size_t ib_cmrc_xfer_window (struct ib_cmrc_connection *cmrc)
 Check CMRC flow control window. More...
 
static struct deviceib_cmrc_identify_device (struct ib_cmrc_connection *cmrc)
 Identify device underlying CMRC connection. More...
 
int ib_cmrc_open (struct interface *xfer, struct ib_device *ibdev, union ib_gid *dgid, union ib_guid *service_id, const char *name)
 Open CMRC connection. More...
 

Variables

static struct ib_connection_operations ib_cmrc_conn_op
 CMRC connection operations. More...
 
static struct ib_completion_queue_operations ib_cmrc_completion_ops
 Infiniband CMRC completion operations. More...
 
static struct ib_queue_pair_operations ib_cmrc_queue_pair_ops
 Infiniband CMRC queue pair operations. More...
 
static struct interface_operation ib_cmrc_xfer_operations []
 CMRC data transfer interface operations. More...
 
static struct interface_descriptor ib_cmrc_xfer_desc
 CMRC data transfer interface descriptor. More...
 
static struct process_descriptor ib_cmrc_shutdown_desc
 CMRC shutdown process descriptor. More...
 

Detailed Description

Infiniband Communication-managed Reliable Connections.

Definition in file ib_cmrc.c.

Macro Definition Documentation

◆ IB_CMRC_NUM_SEND_WQES

#define IB_CMRC_NUM_SEND_WQES   4

CMRC number of send WQEs.

This is a policy decision.

Definition at line 54 of file ib_cmrc.c.

◆ IB_CMRC_NUM_RECV_WQES

#define IB_CMRC_NUM_RECV_WQES   2

CMRC number of receive WQEs.

This is a policy decision.

Definition at line 60 of file ib_cmrc.c.

◆ IB_CMRC_NUM_CQES

#define IB_CMRC_NUM_CQES   8

CMRC number of completion queue entries.

This is a policy decision

Definition at line 66 of file ib_cmrc.c.

Function Documentation

◆ FILE_LICENCE()

FILE_LICENCE ( BSD2  )

◆ ib_cmrc_shutdown()

static void ib_cmrc_shutdown ( struct ib_cmrc_connection cmrc)
static

Shut down CMRC connection gracefully.

Parameters
cmrcCommunication-Managed Reliable Connection

The Infiniband data structures are not reference-counted or guarded. It is therefore unsafe to shut them down while we may be in the middle of a callback from the Infiniband stack (e.g. in a receive completion handler).

This shutdown process will run some time after the call to ib_cmrc_close(), after control has returned out of the Infiniband core, and will shut down the Infiniband interfaces cleanly.

The shutdown process holds an implicit reference on the CMRC connection, ensuring that the structure is not freed before the shutdown process has run.

Definition at line 112 of file ib_cmrc.c.

112  {
113  struct ib_device *ibdev = cmrc->ibdev;
114 
115  DBGC ( cmrc, "CMRC %s %s shutting down\n",
116  ibdev->name, cmrc->name );
117 
118  /* Shut down Infiniband interface */
119  ib_destroy_conn ( ibdev, cmrc->qp, cmrc->conn );
120  ib_destroy_qp ( ibdev, cmrc->qp );
121  ib_destroy_cq ( ibdev, cmrc->cq );
122  ib_close ( ibdev );
123 
124  /* Cancel any pending shutdown */
125  process_del ( &cmrc->shutdown );
126 
127  /* Drop the remaining reference */
128  ref_put ( &cmrc->refcnt );
129 }
struct ib_completion_queue * cq
Completion queue.
Definition: ib_cmrc.c:79
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
#define DBGC(...)
Definition: compiler.h:505
void ib_close(struct ib_device *ibdev)
Close port.
Definition: infiniband.c:716
struct process shutdown
Shutdown process.
Definition: ib_cmrc.c:91
void process_del(struct process *process)
Remove process from process list.
Definition: process.c:79
void ib_destroy_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Destroy completion queue.
Definition: infiniband.c:145
const char * name
Name.
Definition: ib_cmrc.c:73
An Infiniband device.
Definition: infiniband.h:398
void ib_destroy_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Destroy queue pair.
Definition: infiniband.c:314
struct refcnt refcnt
Reference count.
Definition: ib_cmrc.c:71
struct ib_connection * conn
Connection.
Definition: ib_cmrc.c:83
void ib_destroy_conn(struct ib_device *ibdev, struct ib_queue_pair *qp __unused, struct ib_connection *conn)
Destroy connection to remote QP.
Definition: ib_cm.c:504
struct ib_queue_pair * qp
Queue pair.
Definition: ib_cmrc.c:81
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77
#define ref_put(refcnt)
Drop reference to object.
Definition: refcnt.h:106

References ib_cmrc_connection::conn, ib_cmrc_connection::cq, DBGC, ib_close(), ib_destroy_conn(), ib_destroy_cq(), ib_destroy_qp(), ib_cmrc_connection::ibdev, ib_cmrc_connection::name, ib_device::name, process_del(), ib_cmrc_connection::qp, ref_put, ib_cmrc_connection::refcnt, and ib_cmrc_connection::shutdown.

◆ ib_cmrc_close()

static void ib_cmrc_close ( struct ib_cmrc_connection cmrc,
int  rc 
)
static

Close CMRC connection.

Parameters
cmrcCommunication-Managed Reliable Connection
rcReason for close

Definition at line 137 of file ib_cmrc.c.

137  {
138 
139  /* Close data transfer interface */
140  intf_shutdown ( &cmrc->xfer, rc );
141 
142  /* Schedule shutdown process */
143  process_add ( &cmrc->shutdown );
144 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
void intf_shutdown(struct interface *intf, int rc)
Shut down an object interface.
Definition: interface.c:278
struct process shutdown
Shutdown process.
Definition: ib_cmrc.c:91
void process_add(struct process *process)
Add process to process list.
Definition: process.c:59
struct interface xfer
Data transfer interface.
Definition: ib_cmrc.c:75

References intf_shutdown(), process_add(), rc, ib_cmrc_connection::shutdown, and ib_cmrc_connection::xfer.

Referenced by ib_cmrc_changed(), ib_cmrc_complete_recv(), ib_cmrc_complete_send(), and ib_cmrc_xfer_deliver().

◆ ib_cmrc_changed()

static void ib_cmrc_changed ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct ib_connection *conn  __unused,
int  rc_cm,
void *  private_data,
size_t  private_data_len 
)
static

Handle change of CMRC connection status.

Parameters
ibdevInfiniband device
qpQueue pair
connConnection
rc_cmConnection status code
private_dataPrivate data, if available
private_data_lenLength of private data

Definition at line 156 of file ib_cmrc.c.

159  {
160  struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
161  int rc_xfer;
162 
163  /* Record connection status */
164  if ( rc_cm == 0 ) {
165  DBGC ( cmrc, "CMRC %s %s connected\n",
166  ibdev->name, cmrc->name );
167  cmrc->connected = 1;
168  } else {
169  DBGC ( cmrc, "CMRC %s %s disconnected: %s\n",
170  ibdev->name, cmrc->name, strerror ( rc_cm ) );
171  cmrc->connected = 0;
172  }
173 
174  /* Pass up any private data */
175  DBGC2 ( cmrc, "CMRC %s %s received private data:\n",
176  ibdev->name, cmrc->name );
177  DBGC2_HDA ( cmrc, 0, private_data, private_data_len );
178  if ( private_data &&
179  ( rc_xfer = xfer_deliver_raw ( &cmrc->xfer, private_data,
180  private_data_len ) ) != 0 ) {
181  DBGC ( cmrc, "CMRC %s %s could not deliver private data: %s\n",
182  ibdev->name, cmrc->name, strerror ( rc_xfer ) );
183  ib_cmrc_close ( cmrc, rc_xfer );
184  return;
185  }
186 
187  /* Notify upper connection of window change */
188  xfer_window_changed ( &cmrc->xfer );
189 
190  /* If we are disconnected, close the upper connection */
191  if ( rc_cm != 0 ) {
192  ib_cmrc_close ( cmrc, rc_cm );
193  return;
194  }
195 }
void xfer_window_changed(struct interface *intf)
Report change of flow control window.
Definition: xfer.c:146
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
#define DBGC(...)
Definition: compiler.h:505
int xfer_deliver_raw(struct interface *intf, const void *data, size_t len)
Deliver datagram as raw data without metadata.
Definition: xfer.c:288
const char * name
Name.
Definition: ib_cmrc.c:73
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
#define DBGC2_HDA(...)
Definition: compiler.h:523
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static __always_inline void * ib_qp_get_ownerdata(struct ib_queue_pair *qp)
Get Infiniband queue pair owner-private data.
Definition: infiniband.h:664
int connected
QP is connected.
Definition: ib_cmrc.c:89
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
struct interface xfer
Data transfer interface.
Definition: ib_cmrc.c:75
#define DBGC2(...)
Definition: compiler.h:522
static void ib_cmrc_close(struct ib_cmrc_connection *cmrc, int rc)
Close CMRC connection.
Definition: ib_cmrc.c:137
uint8_t private_data[92]
Private data.
Definition: ib_mad.h:44
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77

References ib_cmrc_connection::connected, DBGC, DBGC2, DBGC2_HDA, ib_cmrc_close(), ib_qp_get_ownerdata(), ib_cmrc_connection::ibdev, ib_cmrc_connection::name, ib_device::name, private_data, qp, strerror(), ib_cmrc_connection::xfer, xfer_deliver_raw(), and xfer_window_changed().

◆ ib_cmrc_complete_send()

static void ib_cmrc_complete_send ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct io_buffer iobuf,
int  rc 
)
static

Handle CMRC send completion.

Parameters
ibdevInfiniband device
qpQueue pair
iobufI/O buffer
rcCompletion status code

Definition at line 210 of file ib_cmrc.c.

212  {
213  struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
214 
215  /* Free the completed I/O buffer */
216  free_iob ( iobuf );
217 
218  /* Close the connection on any send errors */
219  if ( rc != 0 ) {
220  DBGC ( cmrc, "CMRC %s %s send error: %s\n",
221  ibdev->name, cmrc->name, strerror ( rc ) );
222  ib_cmrc_close ( cmrc, rc );
223  return;
224  }
225 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
#define DBGC(...)
Definition: compiler.h:505
const char * name
Name.
Definition: ib_cmrc.c:73
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static __always_inline void * ib_qp_get_ownerdata(struct ib_queue_pair *qp)
Get Infiniband queue pair owner-private data.
Definition: infiniband.h:664
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
static void ib_cmrc_close(struct ib_cmrc_connection *cmrc, int rc)
Close CMRC connection.
Definition: ib_cmrc.c:137
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77

References DBGC, free_iob(), ib_cmrc_close(), ib_qp_get_ownerdata(), ib_cmrc_connection::ibdev, ib_cmrc_connection::name, ib_device::name, qp, rc, and strerror().

◆ ib_cmrc_complete_recv()

static void ib_cmrc_complete_recv ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct ib_address_vector *dest  __unused,
struct ib_address_vector *source  __unused,
struct io_buffer iobuf,
int  rc 
)
static

Handle CMRC receive completion.

Parameters
ibdevInfiniband device
qpQueue pair
destDestination address vector, or NULL
sourceSource address vector, or NULL
iobufI/O buffer
rcCompletion status code

Definition at line 237 of file ib_cmrc.c.

241  {
242  struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
243 
244  /* Close the connection on any receive errors */
245  if ( rc != 0 ) {
246  DBGC ( cmrc, "CMRC %s %s receive error: %s\n",
247  ibdev->name, cmrc->name, strerror ( rc ) );
248  free_iob ( iobuf );
249  ib_cmrc_close ( cmrc, rc );
250  return;
251  }
252 
253  DBGC2 ( cmrc, "CMRC %s %s received:\n", ibdev->name, cmrc->name );
254  DBGC2_HDA ( cmrc, 0, iobuf->data, iob_len ( iobuf ) );
255 
256  /* Pass up data */
257  if ( ( rc = xfer_deliver_iob ( &cmrc->xfer, iobuf ) ) != 0 ) {
258  DBGC ( cmrc, "CMRC %s %s could not deliver data: %s\n",
259  ibdev->name, cmrc->name, strerror ( rc ) );
260  ib_cmrc_close ( cmrc, rc );
261  return;
262  }
263 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
int xfer_deliver_iob(struct interface *intf, struct io_buffer *iobuf)
Deliver datagram as I/O buffer without metadata.
Definition: xfer.c:255
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
#define DBGC(...)
Definition: compiler.h:505
const char * name
Name.
Definition: ib_cmrc.c:73
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
#define DBGC2_HDA(...)
Definition: compiler.h:523
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static __always_inline void * ib_qp_get_ownerdata(struct ib_queue_pair *qp)
Get Infiniband queue pair owner-private data.
Definition: infiniband.h:664
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
struct interface xfer
Data transfer interface.
Definition: ib_cmrc.c:75
#define DBGC2(...)
Definition: compiler.h:522
static void ib_cmrc_close(struct ib_cmrc_connection *cmrc, int rc)
Close CMRC connection.
Definition: ib_cmrc.c:137
void * data
Start of data.
Definition: iobuf.h:48
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77

References io_buffer::data, DBGC, DBGC2, DBGC2_HDA, free_iob(), ib_cmrc_close(), ib_qp_get_ownerdata(), ib_cmrc_connection::ibdev, iob_len(), ib_cmrc_connection::name, ib_device::name, qp, rc, strerror(), ib_cmrc_connection::xfer, and xfer_deliver_iob().

◆ ib_cmrc_xfer_deliver()

static int ib_cmrc_xfer_deliver ( struct ib_cmrc_connection cmrc,
struct io_buffer iobuf,
struct xfer_metadata *meta  __unused 
)
static

Send data via CMRC.

Parameters
cmrcCMRC connection
iobufDatagram I/O buffer
metaData transfer metadata
Return values
rcReturn status code

Definition at line 284 of file ib_cmrc.c.

286  {
287  struct ib_device *ibdev = cmrc->ibdev;
288  int rc;
289 
290  /* If no connection has yet been attempted, send this datagram
291  * as the CM REQ private data. Otherwise, send it via the QP.
292  */
293  if ( ! cmrc->connected ) {
294 
295  /* Abort if we have already sent a CM connection request */
296  if ( cmrc->conn ) {
297  DBGC ( cmrc, "CMRC %s %s attempt to send before "
298  "connection is complete\n",
299  ibdev->name, cmrc->name );
300  rc = -EIO;
301  goto out;
302  }
303 
304  /* Send via CM connection request */
305  cmrc->conn = ib_create_conn ( cmrc->ibdev, cmrc->qp,
306  &cmrc->dgid, &cmrc->service_id,
307  iobuf->data, iob_len ( iobuf ),
308  &ib_cmrc_conn_op );
309  if ( ! cmrc->conn ) {
310  DBGC ( cmrc, "CMRC %s %s could not connect\n",
311  ibdev->name, cmrc->name );
312  rc = -ENOMEM;
313  goto out;
314  }
315  DBGC ( cmrc, "CMRC %s %s using CM %08x\n",
316  ibdev->name, cmrc->name, cmrc->conn->local_id );
317 
318  } else {
319 
320  /* Send via QP */
321  if ( ( rc = ib_post_send ( cmrc->ibdev, cmrc->qp, NULL,
322  iob_disown ( iobuf ) ) ) != 0 ) {
323  DBGC ( cmrc, "CMRC %s %s could not send: %s\n",
324  ibdev->name, cmrc->name, strerror ( rc ) );
325  goto out;
326  }
327 
328  }
329  return 0;
330 
331  out:
332  /* Free the I/O buffer if necessary */
333  free_iob ( iobuf );
334 
335  /* Close the connection on any errors */
336  if ( rc != 0 )
337  ib_cmrc_close ( cmrc, rc );
338 
339  return rc;
340 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
#define DBGC(...)
Definition: compiler.h:505
struct ib_connection * ib_create_conn(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *dgid, union ib_guid *service_id, void *private_data, size_t private_data_len, struct ib_connection_operations *op)
Create connection to remote QP.
Definition: ib_cm.c:451
union ib_gid dgid
Destination GID.
Definition: ib_cmrc.c:85
__be32 out[4]
Definition: CIB_PRM.h:36
const char * name
Name.
Definition: ib_cmrc.c:73
uint32_t local_id
Local communication ID.
Definition: ib_cm.h:41
An Infiniband device.
Definition: infiniband.h:398
#define ENOMEM
Not enough space.
Definition: errno.h:534
#define iob_disown(iobuf)
Disown an I/O buffer.
Definition: iobuf.h:212
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
int connected
QP is connected.
Definition: ib_cmrc.c:89
union ib_guid service_id
Service ID.
Definition: ib_cmrc.c:87
int ib_post_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf)
Post send work queue entry.
Definition: infiniband.c:416
struct ib_connection * conn
Connection.
Definition: ib_cmrc.c:83
static void ib_cmrc_close(struct ib_cmrc_connection *cmrc, int rc)
Close CMRC connection.
Definition: ib_cmrc.c:137
void * data
Start of data.
Definition: iobuf.h:48
#define EIO
Input/output error.
Definition: errno.h:433
static struct ib_connection_operations ib_cmrc_conn_op
CMRC connection operations.
Definition: ib_cmrc.c:198
struct ib_queue_pair * qp
Queue pair.
Definition: ib_cmrc.c:81
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77

References ib_cmrc_connection::conn, ib_cmrc_connection::connected, io_buffer::data, DBGC, ib_cmrc_connection::dgid, EIO, ENOMEM, free_iob(), ib_cmrc_close(), ib_cmrc_conn_op, ib_create_conn(), ib_post_send(), ib_cmrc_connection::ibdev, iob_disown, iob_len(), ib_connection::local_id, ib_cmrc_connection::name, ib_device::name, NULL, out, ib_cmrc_connection::qp, rc, ib_cmrc_connection::service_id, and strerror().

◆ ib_cmrc_xfer_window()

static size_t ib_cmrc_xfer_window ( struct ib_cmrc_connection cmrc)
static

Check CMRC flow control window.

Parameters
cmrcCMRC connection
Return values
lenLength of window

Definition at line 348 of file ib_cmrc.c.

348  {
349 
350  /* We indicate a window only when we are successfully
351  * connected.
352  */
353  return ( cmrc->connected ? IB_MAX_PAYLOAD_SIZE : 0 );
354 }
int connected
QP is connected.
Definition: ib_cmrc.c:89
#define IB_MAX_PAYLOAD_SIZE
Maximum payload size.
Definition: infiniband.h:50

References ib_cmrc_connection::connected, and IB_MAX_PAYLOAD_SIZE.

◆ ib_cmrc_identify_device()

static struct device* ib_cmrc_identify_device ( struct ib_cmrc_connection cmrc)
static

Identify device underlying CMRC connection.

Parameters
cmrcCMRC connection
Return values
deviceUnderlying device

Definition at line 363 of file ib_cmrc.c.

363  {
364  return cmrc->ibdev->dev;
365 }
struct device * dev
Underlying device.
Definition: infiniband.h:410
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77

References ib_device::dev, and ib_cmrc_connection::ibdev.

◆ ib_cmrc_open()

int ib_cmrc_open ( struct interface xfer,
struct ib_device ibdev,
union ib_gid dgid,
union ib_guid service_id,
const char *  name 
)

Open CMRC connection.

Parameters
xferData transfer interface
ibdevInfiniband device
dgidDestination GID
service_idService ID
nameConnection name
Return values
rcReturns status code

Definition at line 397 of file ib_cmrc.c.

399  {
400  struct ib_cmrc_connection *cmrc;
401  int rc;
402 
403  /* Allocate and initialise structure */
404  cmrc = zalloc ( sizeof ( *cmrc ) );
405  if ( ! cmrc ) {
406  rc = -ENOMEM;
407  goto err_alloc;
408  }
409  ref_init ( &cmrc->refcnt, NULL );
410  cmrc->name = name;
411  intf_init ( &cmrc->xfer, &ib_cmrc_xfer_desc, &cmrc->refcnt );
412  cmrc->ibdev = ibdev;
413  memcpy ( &cmrc->dgid, dgid, sizeof ( cmrc->dgid ) );
414  memcpy ( &cmrc->service_id, service_id, sizeof ( cmrc->service_id ) );
416  &cmrc->refcnt );
417 
418  /* Open Infiniband device */
419  if ( ( rc = ib_open ( ibdev ) ) != 0 ) {
420  DBGC ( cmrc, "CMRC %s %s could not open device: %s\n",
421  ibdev->name, cmrc->name, strerror ( rc ) );
422  goto err_open;
423  }
424 
425  /* Create completion queue */
427  &ib_cmrc_completion_ops, &cmrc->cq ) ) != 0){
428  DBGC ( cmrc, "CMRC %s %s could not create completion queue: "
429  "%s\n", ibdev->name, cmrc->name, strerror ( rc ) );
430  goto err_create_cq;
431  }
432 
433  /* Create queue pair */
435  cmrc->cq, IB_CMRC_NUM_RECV_WQES, cmrc->cq,
437  &cmrc->qp ) ) != 0 ) {
438  DBGC ( cmrc, "CMRC %s %s could not create queue pair: %s\n",
439  ibdev->name, cmrc->name, strerror ( rc ) );
440  goto err_create_qp;
441  }
442  ib_qp_set_ownerdata ( cmrc->qp, cmrc );
443  DBGC ( cmrc, "CMRC %s %s using QPN %#lx\n",
444  ibdev->name, cmrc->name, cmrc->qp->qpn );
445 
446  /* Attach to parent interface, transfer reference (implicitly)
447  * to our shutdown process, and return.
448  */
449  intf_plug_plug ( &cmrc->xfer, xfer );
450  return 0;
451 
452  ib_destroy_qp ( ibdev, cmrc->qp );
453  err_create_qp:
454  ib_destroy_cq ( ibdev, cmrc->cq );
455  err_create_cq:
456  ib_close ( ibdev );
457  err_open:
458  ref_put ( &cmrc->refcnt );
459  err_alloc:
460  return rc;
461 }
struct ib_completion_queue * cq
Completion queue.
Definition: ib_cmrc.c:79
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
const char * name
Definition: ath9k_hw.c:1984
static struct interface_descriptor ib_cmrc_xfer_desc
CMRC data transfer interface descriptor.
Definition: ib_cmrc.c:379
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
Definition: infiniband.h:408
#define ref_init(refcnt, free)
Initialise a reference counter.
Definition: refcnt.h:64
int ib_create_cq(struct ib_device *ibdev, unsigned int num_cqes, struct ib_completion_queue_operations *op, struct ib_completion_queue **new_cq)
Create completion queue.
Definition: infiniband.c:98
#define DBGC(...)
Definition: compiler.h:505
static struct ib_queue_pair_operations ib_cmrc_queue_pair_ops
Infiniband CMRC queue pair operations.
Definition: ib_cmrc.c:272
int ib_open(struct ib_device *ibdev)
Open port.
Definition: infiniband.c:652
void intf_plug_plug(struct interface *a, struct interface *b)
Plug two object interfaces together.
Definition: interface.c:107
static struct process_descriptor ib_cmrc_shutdown_desc
CMRC shutdown process descriptor.
Definition: ib_cmrc.c:383
void ib_close(struct ib_device *ibdev)
Close port.
Definition: infiniband.c:716
struct process shutdown
Shutdown process.
Definition: ib_cmrc.c:91
union ib_gid dgid
Destination GID.
Definition: ib_cmrc.c:85
void ib_destroy_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Destroy completion queue.
Definition: infiniband.c:145
const char * name
Name.
Definition: ib_cmrc.c:73
#define ENOMEM
Not enough space.
Definition: errno.h:534
#define IB_CMRC_NUM_RECV_WQES
CMRC number of receive WQEs.
Definition: ib_cmrc.c:60
void * memcpy(void *dest, const void *src, size_t len) __nonnull
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
unsigned long qpn
Queue pair number.
Definition: infiniband.h:165
#define IB_CMRC_NUM_SEND_WQES
CMRC number of send WQEs.
Definition: ib_cmrc.c:54
#define IB_CMRC_NUM_CQES
CMRC number of completion queue entries.
Definition: ib_cmrc.c:66
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
void ib_destroy_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Destroy queue pair.
Definition: infiniband.c:314
static void process_init_stopped(struct process *process, struct process_descriptor *desc, struct refcnt *refcnt)
Initialise process without adding to process list.
Definition: process.h:145
static struct ib_completion_queue_operations ib_cmrc_completion_ops
Infiniband CMRC completion operations.
Definition: ib_cmrc.c:266
union ib_guid service_id
Service ID.
Definition: ib_cmrc.c:87
union ib_gid dgid
Definition: ib_mad.h:12
struct interface xfer
Data transfer interface.
Definition: ib_cmrc.c:75
struct refcnt refcnt
Reference count.
Definition: ib_cmrc.c:71
int ib_create_qp(struct ib_device *ibdev, enum ib_queue_pair_type type, unsigned int num_send_wqes, struct ib_completion_queue *send_cq, unsigned int num_recv_wqes, struct ib_completion_queue *recv_cq, struct ib_queue_pair_operations *op, const char *name, struct ib_queue_pair **new_qp)
Create queue pair.
Definition: infiniband.c:199
struct ib_queue_pair * qp
Queue pair.
Definition: ib_cmrc.c:81
union ib_guid service_id
Service ID.
Definition: ib_mad.h:16
static void intf_init(struct interface *intf, struct interface_descriptor *desc, struct refcnt *refcnt)
Initialise an object interface.
Definition: interface.h:203
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
static __always_inline void ib_qp_set_ownerdata(struct ib_queue_pair *qp, void *priv)
Set Infiniband queue pair owner-private data.
Definition: infiniband.h:653
struct ib_device * ibdev
Infiniband device.
Definition: ib_cmrc.c:77
#define ref_put(refcnt)
Drop reference to object.
Definition: refcnt.h:106

References ib_cmrc_connection::cq, DBGC, dgid, ib_cmrc_connection::dgid, ENOMEM, ib_close(), ib_cmrc_completion_ops, IB_CMRC_NUM_CQES, IB_CMRC_NUM_RECV_WQES, IB_CMRC_NUM_SEND_WQES, ib_cmrc_queue_pair_ops, ib_cmrc_shutdown_desc, ib_cmrc_xfer_desc, ib_create_cq(), ib_create_qp(), ib_destroy_cq(), ib_destroy_qp(), ib_open(), ib_qp_set_ownerdata(), IB_QPT_RC, ib_cmrc_connection::ibdev, intf_init(), intf_plug_plug(), memcpy(), ib_cmrc_connection::name, ib_device::name, name, NULL, process_init_stopped(), ib_cmrc_connection::qp, ib_queue_pair::qpn, rc, ref_init, ref_put, ib_cmrc_connection::refcnt, service_id, ib_cmrc_connection::service_id, ib_cmrc_connection::shutdown, strerror(), ib_cmrc_connection::xfer, and zalloc().

Referenced by ib_srp_open(), and xcm_reopen().

Variable Documentation

◆ ib_cmrc_conn_op

struct ib_connection_operations ib_cmrc_conn_op
static
Initial value:
= {
.changed = ib_cmrc_changed,
}
static void ib_cmrc_changed(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_connection *conn __unused, int rc_cm, void *private_data, size_t private_data_len)
Handle change of CMRC connection status.
Definition: ib_cmrc.c:156

CMRC connection operations.

Definition at line 198 of file ib_cmrc.c.

Referenced by ib_cmrc_xfer_deliver().

◆ ib_cmrc_completion_ops

struct ib_completion_queue_operations ib_cmrc_completion_ops
static
Initial value:
= {
.complete_send = ib_cmrc_complete_send,
.complete_recv = ib_cmrc_complete_recv,
}
static void ib_cmrc_complete_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf, int rc)
Handle CMRC send completion.
Definition: ib_cmrc.c:210
static void ib_cmrc_complete_recv(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest __unused, struct ib_address_vector *source __unused, struct io_buffer *iobuf, int rc)
Handle CMRC receive completion.
Definition: ib_cmrc.c:237

Infiniband CMRC completion operations.

Definition at line 266 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

◆ ib_cmrc_queue_pair_ops

struct ib_queue_pair_operations ib_cmrc_queue_pair_ops
static
Initial value:
= {
.alloc_iob = alloc_iob,
}
struct io_buffer * alloc_iob(size_t len)
Allocate I/O buffer.
Definition: iobuf.c:129

Infiniband CMRC queue pair operations.

Definition at line 272 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

◆ ib_cmrc_xfer_operations

struct interface_operation ib_cmrc_xfer_operations[]
static
Initial value:
= {
}
void intf_close(struct interface *intf, int rc)
Close an object interface.
Definition: interface.c:249
static struct device * ib_cmrc_identify_device(struct ib_cmrc_connection *cmrc)
Identify device underlying CMRC connection.
Definition: ib_cmrc.c:363
static int ib_cmrc_xfer_deliver(struct ib_cmrc_connection *cmrc, struct io_buffer *iobuf, struct xfer_metadata *meta __unused)
Send data via CMRC.
Definition: ib_cmrc.c:284
static size_t ib_cmrc_xfer_window(struct ib_cmrc_connection *cmrc)
Check CMRC flow control window.
Definition: ib_cmrc.c:348
size_t xfer_window(struct interface *intf)
Check flow control window.
Definition: xfer.c:116
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
#define INTF_OP(op_type, object_type, op_func)
Define an object interface operation.
Definition: interface.h:32
int xfer_deliver(struct interface *intf, struct io_buffer *iobuf, struct xfer_metadata *meta)
Deliver datagram.
Definition: xfer.c:194
static void ib_cmrc_close(struct ib_cmrc_connection *cmrc, int rc)
Close CMRC connection.
Definition: ib_cmrc.c:137
struct device * identify_device(struct interface *intf)
Identify a device behind an interface.
Definition: device.c:125

CMRC data transfer interface operations.

Definition at line 368 of file ib_cmrc.c.

◆ ib_cmrc_xfer_desc

struct interface_descriptor ib_cmrc_xfer_desc
static
Initial value:
=
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
static struct interface_operation ib_cmrc_xfer_operations[]
CMRC data transfer interface operations.
Definition: ib_cmrc.c:368
#define INTF_DESC(object_type, intf, operations)
Define an object interface descriptor.
Definition: interface.h:80
struct interface xfer
Data transfer interface.
Definition: ib_cmrc.c:75

CMRC data transfer interface descriptor.

Definition at line 379 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

◆ ib_cmrc_shutdown_desc

struct process_descriptor ib_cmrc_shutdown_desc
static
Initial value:
=
#define PROC_DESC_ONCE(object_type, process, _step)
Define a process descriptor for a process that runs only once.
Definition: process.h:97
An Infiniband Communication-Managed Reliable Connection.
Definition: ib_cmrc.c:69
static void ib_cmrc_shutdown(struct ib_cmrc_connection *cmrc)
Shut down CMRC connection gracefully.
Definition: ib_cmrc.c:112
void shutdown(int flags)
Shut down iPXE.
Definition: init.c:98

CMRC shutdown process descriptor.

Definition at line 383 of file ib_cmrc.c.

Referenced by ib_cmrc_open().