ib_mi.c File Reference

Infiniband management interfaces. More...

#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <byteswap.h>
#include <ipxe/infiniband.h>
#include <ipxe/iobuf.h>
#include <ipxe/ib_mi.h>

Go to the source code of this file.

Macros
#define	IB_MI_NUM_SEND_WQES   4
	Management interface number of send WQEs.
#define	IB_MI_NUM_RECV_WQES   2
	Management interface number of receive WQEs.
#define	IB_MI_NUM_CQES   8
	Management interface number of completion queue entries.
#define	IB_MI_TID_MAGIC   ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
	TID magic signature.

Functions
	FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
static int	ib_mi_handle (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
	Handle received MAD.
static void	ib_mi_complete_recv (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest __unused, struct ib_address_vector *source, struct io_buffer *iobuf, int rc)
	Complete receive via management interface.
int	ib_mi_send (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
	Transmit MAD.
static void	ib_mi_timer_expired (struct retry_timer *timer, int expired)
	Handle management transaction timer expiry.
struct ib_mad_transaction *	ib_create_madx (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av, struct ib_mad_transaction_operations *op)
	Create management transaction.
void	ib_destroy_madx (struct ib_device *ibdev __unused, struct ib_mad_interface *mi __unused, struct ib_mad_transaction *madx)
	Destroy management transaction.
int	ib_create_mi (struct ib_device *ibdev, enum ib_queue_pair_type type, struct ib_mad_interface **new_mi)
	Create management interface.
void	ib_destroy_mi (struct ib_device *ibdev, struct ib_mad_interface *mi)
	Destroy management interface.

Variables
static unsigned int	next_tid
	TID to use for next MAD.
static struct ib_completion_queue_operations	ib_mi_completion_ops
	Management interface completion operations.
static struct ib_queue_pair_operations	ib_mi_queue_pair_ops
	Management interface queue pair operations.

Detailed Description

Infiniband management interfaces.

Definition in file ib_mi.c.

Macro Definition Documentation

IB_MI_NUM_SEND_WQES

#define IB_MI_NUM_SEND_WQES   4

Management interface number of send WQEs.

This is a policy decision.

Definition at line 48 of file ib_mi.c.

Referenced by ib_create_mi().

IB_MI_NUM_RECV_WQES

#define IB_MI_NUM_RECV_WQES   2

Management interface number of receive WQEs.

This is a policy decision.

Definition at line 54 of file ib_mi.c.

Referenced by ib_create_mi().

IB_MI_NUM_CQES

#define IB_MI_NUM_CQES   8

Management interface number of completion queue entries.

This is a policy decision

Definition at line 60 of file ib_mi.c.

Referenced by ib_create_mi().

IB_MI_TID_MAGIC

#define IB_MI_TID_MAGIC   ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )

TID magic signature.

Definition at line 63 of file ib_mi.c.

Referenced by ib_mi_send().

Function Documentation

FILE_LICENCE()

FILE_LICENCE

(

GPL2_OR_LATER_OR_UBDL

)

ib_mi_handle()

int ib_mi_handle ( struct ib_device * ibdev, struct ib_mad_interface * mi, union ib_mad * mad, struct ib_address_vector * av )

static

Handle received MAD.

Parameters

ibdev	Infiniband device
mi	Management interface
mad	Received MAD
av	Source address vector

Return values

rc	Return status code

Definition at line 77 of file ib_mi.c.

                                                         {
        struct ib_mad_hdr *hdr = &mad->hdr;
        struct ib_mad_transaction *madx;
        struct ib_mad_agent *agent;
 
        /* Look for a matching transaction by TID */
        list_for_each_entry ( madx, &mi->madx, list ) {
                if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
                              sizeof ( hdr->tid ) ) != 0 )
                        continue;
                /* Found a matching transaction */
                madx->op->complete ( ibdev, mi, madx, 0, mad, av );
                return 0;
        }
 
        /* If there is no matching transaction, look for a listening agent */
        for_each_table_entry ( agent, IB_MAD_AGENTS ) {
                if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
                       ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
                     ( agent->class_version != hdr->class_version ) ||
                     ( agent->attr_id != hdr->attr_id ) )
                        continue;
                /* Found a matching agent */
                agent->handle ( ibdev, mi, mad, av );
                return 0;
        }
 
        /* Otherwise, ignore it */
        DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
               mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
        return -ENOTSUP;
}

References ib_mad_agent::attr_id, ib_mad_agent::class_version, ib_mad_transaction_operations::complete, DBGC, ENOTSUP, for_each_table_entry, ib_mad_agent::handle, hdr, ib_mad::hdr, IB_MAD_AGENTS, IB_MGMT_CLASS_MASK, list_for_each_entry, ib_mad_transaction::mad, mad, ib_mad_interface::madx, memcmp(), ib_mad_agent::mgmt_class, ntohl, ib_mad_transaction::op, and ib_mad_hdr::tid.

Referenced by ib_mi_complete_recv().

ib_mi_complete_recv()

void ib_mi_complete_recv ( struct ib_device * ibdev, struct ib_queue_pair * qp, struct ib_address_vector *dest __unused, struct ib_address_vector * source, struct io_buffer * iobuf, int rc )

static

Complete receive via management interface.

Parameters

ibdev	Infiniband device
qp	Queue pair
dest	Destination address vector
source	Source address vector
iobuf	I/O buffer
rc	Completion status code

Definition at line 124 of file ib_mi.c.

                                                                    {
        struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
        union ib_mad *mad;
        struct ib_mad_hdr *hdr;
 
        /* Ignore errors */
        if ( rc != 0 ) {
                DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
                goto out;
        }
 
        /* Sanity checks */
        if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
                DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
                       mi, iob_len ( iobuf ) );
                DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
                goto out;
        }
        mad = iobuf->data;
        hdr = &mad->hdr;
        if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
                DBGC ( mi, "MI %p RX unsupported base version %x\n",
                       mi, hdr->base_version );
                DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
                goto out;
        }
        DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
               "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
               hdr->mgmt_class, hdr->class_version, hdr->method,
               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
 
        /* Handle MAD */
        if ( ( rc = ib_mi_handle ( ibdev, mi, mad, source ) ) != 0 )
                goto out;
 
 out:
        free_iob ( iobuf );
}

References __unused, io_buffer::data, DBGC, DBGC2_HDA, DBGC_HDA, dest, free_iob(), hdr, IB_MGMT_BASE_VERSION, ib_mi_handle(), ib_qp_get_ownerdata(), iob_len(), mad, ntohl, ntohs, out, qp, rc, and strerror().

ib_mi_send()

int ib_mi_send	(	struct ib_device *	ibdev,
		struct ib_mad_interface *	mi,
		union ib_mad *	mad,
		struct ib_address_vector *	av )

Transmit MAD.

Parameters

ibdev	Infiniband device
mi	Management interface
mad	MAD
av	Destination address vector

Return values

rc	Return status code

Definition at line 187 of file ib_mi.c.

                                                                   {
        struct ib_mad_hdr *hdr = &mad->hdr;
        struct io_buffer *iobuf;
        int rc;
 
        /* Set common fields */
        hdr->base_version = IB_MGMT_BASE_VERSION;
        if ( ( hdr->tid.high == 0 ) && ( hdr->tid.low == 0 ) ) {
                hdr->tid.high = htonl ( IB_MI_TID_MAGIC );
                hdr->tid.low = htonl ( ++next_tid );
        }
        DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
               "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
               hdr->mgmt_class, hdr->class_version, hdr->method,
               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
 
        /* Construct directed route portion of response, if necessary */
        if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
                struct ib_mad_smp *smp = &mad->smp;
                unsigned int hop_pointer;
                unsigned int hop_count;
 
                smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
                hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
                hop_count = smp->mad_hdr.class_specific.smp.hop_count;
                assert ( hop_count == hop_pointer );
                if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
                                     sizeof ( smp->return_path.hops[0] ) ) ) {
                        smp->return_path.hops[hop_pointer] = ibdev->port;
                } else {
                        DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
                               "%d\n", mi, ntohl ( hdr->tid.high ),
                               ntohl ( hdr->tid.low ), hop_pointer );
                        return -EINVAL;
                }
        }
 
        /* Construct I/O buffer */
        iobuf = alloc_iob ( sizeof ( *mad ) );
        if ( ! iobuf ) {
                DBGC ( mi, "MI %p could not allocate buffer for TID "
                       "%08x%08x\n",
                       mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
                return -ENOMEM;
        }
        memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );
 
        /* Send I/O buffer */
        if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
                DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
                       mi,  ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
                       strerror ( rc ) );
                free_iob ( iobuf );
                return rc;
        }
 
        return 0;
}

References alloc_iob(), assert, DBGC, DBGC2_HDA, EINVAL, ENOMEM, free_iob(), hdr, hop_count, hop_pointer, htonl, htons, IB_MGMT_BASE_VERSION, IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE, IB_MI_TID_MAGIC, ib_post_send(), IB_SMP_STATUS_D_INBOUND, iob_put, mad, memcpy(), next_tid, ntohl, ntohs, ib_device::port, ib_mad_interface::qp, rc, smp, and strerror().

Referenced by ib_cm_send_drep(), ib_cm_send_rtu(), ib_mcast_leave(), ib_mi_timer_expired(), ib_sma_guid_info(), ib_sma_node_desc(), ib_sma_node_info(), ib_sma_pkey_table(), and ib_sma_port_info().

ib_mi_timer_expired()

void ib_mi_timer_expired ( struct retry_timer * timer, int expired )

static

Handle management transaction timer expiry.

Parameters

timer	Retry timer
expired	Failure indicator

Definition at line 254 of file ib_mi.c.

                                                                           {
        struct ib_mad_transaction *madx =
                container_of ( timer, struct ib_mad_transaction, timer );
        struct ib_mad_interface *mi = madx->mi;
        struct ib_device *ibdev = mi->ibdev;
        struct ib_mad_hdr *hdr = &madx->mad.hdr;
 
        /* Abandon transaction if we have tried too many times */
        if ( expired ) {
                DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
                       mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
                madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
                return;
        }
 
        /* Restart retransmission timer */
        start_timer ( timer );
 
        /* Resend MAD */
        ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
}

References ib_mad_transaction::av, ib_mad_transaction_operations::complete, container_of, DBGC, ETIMEDOUT, expired, hdr, ib_mad::hdr, ib_mi_send(), ib_mad_interface::ibdev, ib_mad_transaction::mad, ib_mad_interface::madx, ntohl, NULL, ib_mad_transaction::op, and start_timer().

Referenced by ib_create_madx().

ib_create_madx()

struct ib_mad_transaction * ib_create_madx	(	struct ib_device *	ibdev,
		struct ib_mad_interface *	mi,
		union ib_mad *	mad,
		struct ib_address_vector *	av,
		struct ib_mad_transaction_operations *	op )

Create management transaction.

Parameters

ibdev	Infiniband device
mi	Management interface
mad	MAD to send
av	Destination address, or NULL to use SM's GSI
op	Management transaction operations

Return values

madx	Management transaction, or NULL

Definition at line 287 of file ib_mi.c.

                                                            {
        struct ib_mad_transaction *madx;
 
        /* Allocate and initialise structure */
        madx = zalloc ( sizeof ( *madx ) );
        if ( ! madx )
                return NULL;
        timer_init ( &madx->timer, ib_mi_timer_expired, NULL );
        madx->mi = mi;
        madx->op = op;
 
        /* Determine address vector */
        if ( av ) {
                memcpy ( &madx->av, av, sizeof ( madx->av ) );
        } else {
                madx->av.lid = ibdev->sm_lid;
                madx->av.sl = ibdev->sm_sl;
                madx->av.qpn = IB_QPN_GSI;
                madx->av.qkey = IB_QKEY_GSI;
        }
 
        /* Copy MAD */
        memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );
 
        /* Add to list and start timer to send initial MAD */
        list_add ( &madx->list, &mi->madx );
        start_timer_nodelay ( &madx->timer );
 
        return madx;
}

References ib_mad_transaction::av, ib_mi_timer_expired(), IB_QKEY_GSI, IB_QPN_GSI, ib_address_vector::lid, ib_mad_transaction::list, list_add, ib_mad_transaction::mad, mad, ib_mad_interface::madx, memcpy(), ib_mad_transaction::mi, NULL, ib_mad_transaction::op, op, ib_address_vector::qkey, ib_address_vector::qpn, ib_address_vector::sl, ib_device::sm_lid, ib_device::sm_sl, start_timer_nodelay(), ib_mad_transaction::timer, and zalloc().

Referenced by ib_cm_path_complete(), ib_create_path(), ib_create_service_madx(), ib_mcast_join(), and xsigo_xds_complete().

ib_destroy_madx()

void ib_destroy_madx	(	struct ib_device *ibdev	__unused,
		struct ib_mad_interface *mi	__unused,
		struct ib_mad_transaction *	madx )

Destroy management transaction.

Parameters

ibdev	Infiniband device
mi	Management interface
madx	Management transaction

Definition at line 327 of file ib_mi.c.

                                                         {
 
        /* Stop timer and remove from list */
        stop_timer ( &madx->timer );
        list_del ( &madx->list );
 
        /* Free transaction */
        free ( madx );
}

References __unused, free, ib_mad_transaction::list, list_del, stop_timer(), and ib_mad_transaction::timer.

Referenced by ib_cm_req_complete(), ib_create_path(), ib_destroy_conn(), ib_destroy_path(), ib_mcast_complete(), ib_mcast_join(), ib_mcast_leave(), ib_path_complete(), xsigo_discover(), xsigo_ib_notify(), xsigo_ib_remove(), xsigo_xcm_complete(), and xsigo_xds_complete().

ib_create_mi()

int ib_create_mi	(	struct ib_device *	ibdev,
		enum ib_queue_pair_type	type,
		struct ib_mad_interface **	new_mi )

Create management interface.

Parameters

ibdev	Infiniband device
type	Queue pair type
new_mi	New management interface to fill in

Return values

rc	Return status code

Definition at line 347 of file ib_mi.c.

                                                      {
        struct ib_mad_interface *mi;
        const char *name;
        int rc;
 
        /* Allocate and initialise fields */
        mi = zalloc ( sizeof ( *mi ) );
        if ( ! mi ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        mi->ibdev = ibdev;
        INIT_LIST_HEAD ( &mi->madx );
 
        /* Create completion queue */
        if ( ( rc = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops,
                                   &mi->cq ) ) != 0 ) {
                DBGC ( mi, "MI %p could not create completion queue: %s\n",
                       mi, strerror ( rc ) );
                goto err_create_cq;
        }
 
        /* Create queue pair */
        name = ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" );
        if ( ( rc = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
                                   IB_MI_NUM_RECV_WQES, mi->cq,
                                   &ib_mi_queue_pair_ops, name, &mi->qp ) )!=0){
                DBGC ( mi, "MI %p could not create queue pair: %s\n",
                       mi, strerror ( rc ) );
                goto err_create_qp;
        }
        ib_qp_set_ownerdata ( mi->qp, mi );
        DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
               mi, mi->qp->name, mi->qp->qpn );
 
        /* Set queue key */
        mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
        if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
                DBGC ( mi, "MI %p could not set queue key: %s\n",
                       mi, strerror ( rc ) );
                goto err_modify_qp;
        }
 
        /* Fill receive ring */
        ib_refill_recv ( ibdev, mi->qp );
        *new_mi = mi;
        return 0;
 
 err_modify_qp:
        ib_destroy_qp ( ibdev, mi->qp );
 err_create_qp:
        ib_destroy_cq ( ibdev, mi->cq );
 err_create_cq:
        free ( mi );
 err_alloc:
        return rc;
}

References ib_mad_interface::cq, DBGC, ENOMEM, free, ib_create_cq(), ib_create_qp(), ib_destroy_cq(), ib_destroy_qp(), ib_mi_completion_ops, IB_MI_NUM_CQES, IB_MI_NUM_RECV_WQES, IB_MI_NUM_SEND_WQES, ib_mi_queue_pair_ops, ib_modify_qp(), IB_QKEY_GSI, IB_QKEY_SMI, ib_qp_set_ownerdata(), IB_QPT_SMI, ib_refill_recv(), ib_mad_interface::ibdev, INIT_LIST_HEAD, ib_mad_interface::madx, ib_queue_pair::name, name, ib_queue_pair::qkey, ib_mad_interface::qp, ib_queue_pair::qpn, rc, strerror(), type, and zalloc().

Referenced by ib_open().

ib_destroy_mi()

void ib_destroy_mi	(	struct ib_device *	ibdev,
		struct ib_mad_interface *	mi )

Destroy management interface.

Parameters

mi	Management interface

Definition at line 411 of file ib_mi.c.

                                                                            {
        struct ib_mad_transaction *madx;
        struct ib_mad_transaction *tmp;
 
        /* Flush any outstanding requests */
        list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
                DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
                       mi, ntohl ( madx->mad.hdr.tid.high ),
                       ntohl ( madx->mad.hdr.tid.low ) );
                madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
        }
 
        ib_destroy_qp ( ibdev, mi->qp );
        ib_destroy_cq ( ibdev, mi->cq );
        free ( mi );
}

References ib_mad_transaction_operations::complete, ib_mad_interface::cq, DBGC, ECANCELED, free, ib_mad::hdr, ib_mad_tid::high, ib_destroy_cq(), ib_destroy_qp(), ib_mad_transaction::list, list_for_each_entry_safe, ib_mad_tid::low, ib_mad_transaction::mad, ib_mad_interface::madx, ib_mad_transaction::mi, ntohl, NULL, ib_mad_transaction::op, ib_mad_interface::qp, ib_mad_hdr::tid, and tmp.

Referenced by ib_close(), and ib_open().

Variable Documentation

next_tid

unsigned int next_tid

static

TID to use for next MAD.

Definition at line 66 of file ib_mi.c.

Referenced by ib_mi_send().

ib_mi_completion_ops

struct ib_completion_queue_operations ib_mi_completion_ops

static

Initial value:

= {
        .complete_recv = ib_mi_complete_recv,
}

Management interface completion operations.

Definition at line 169 of file ib_mi.c.

                                                                    {
        .complete_recv = ib_mi_complete_recv,
};

Referenced by ib_create_mi().

ib_mi_queue_pair_ops

struct ib_queue_pair_operations ib_mi_queue_pair_ops

static

Initial value:

= {
        .alloc_iob = alloc_iob,
}

Management interface queue pair operations.

Definition at line 174 of file ib_mi.c.

                                                              {
        .alloc_iob = alloc_iob,
};

Referenced by ib_create_mi().