iPXE
Defines | Functions | Variables
ib_mi.c File Reference

Infiniband management interfaces. More...

#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <byteswap.h>
#include <ipxe/infiniband.h>
#include <ipxe/iobuf.h>
#include <ipxe/ib_mi.h>

Go to the source code of this file.

Defines

#define IB_MI_NUM_SEND_WQES   4
 Management interface number of send WQEs.
#define IB_MI_NUM_RECV_WQES   2
 Management interface number of receive WQEs.
#define IB_MI_NUM_CQES   8
 Management interface number of completion queue entries.
#define IB_MI_TID_MAGIC   ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
 TID magic signature.

Functions

 FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
static int ib_mi_handle (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
 Handle received MAD.
static void ib_mi_complete_recv (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest __unused, struct ib_address_vector *source, struct io_buffer *iobuf, int rc)
 Complete receive via management interface.
int ib_mi_send (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
 Transmit MAD.
static void ib_mi_timer_expired (struct retry_timer *timer, int expired)
 Handle management transaction timer expiry.
struct ib_mad_transactionib_create_madx (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av, struct ib_mad_transaction_operations *op)
 Create management transaction.
void ib_destroy_madx (struct ib_device *ibdev __unused, struct ib_mad_interface *mi __unused, struct ib_mad_transaction *madx)
 Destroy management transaction.
int ib_create_mi (struct ib_device *ibdev, enum ib_queue_pair_type type, struct ib_mad_interface **new_mi)
 Create management interface.
void ib_destroy_mi (struct ib_device *ibdev, struct ib_mad_interface *mi)
 Destroy management interface.

Variables

static unsigned int next_tid
 TID to use for next MAD.
static struct
ib_completion_queue_operations 
ib_mi_completion_ops
 Management interface completion operations.
static struct
ib_queue_pair_operations 
ib_mi_queue_pair_ops
 Management interface queue pair operations.

Detailed Description

Infiniband management interfaces.

Definition in file ib_mi.c.


Define Documentation

#define IB_MI_NUM_SEND_WQES   4

Management interface number of send WQEs.

This is a policy decision.

Definition at line 48 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_NUM_RECV_WQES   2

Management interface number of receive WQEs.

This is a policy decision.

Definition at line 54 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_NUM_CQES   8

Management interface number of completion queue entries.

This is a policy decision

Definition at line 60 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_TID_MAGIC   ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )

TID magic signature.

Definition at line 63 of file ib_mi.c.

Referenced by ib_mi_send().


Function Documentation

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL  )
static int ib_mi_handle ( struct ib_device ibdev,
struct ib_mad_interface mi,
union ib_mad mad,
struct ib_address_vector av 
) [static]

Handle received MAD.

Parameters:
ibdevInfiniband device
miManagement interface
madReceived MAD
avSource address vector
Return values:
rcReturn status code

Definition at line 77 of file ib_mi.c.

References ib_mad_agent::attr_id, ib_mad_hdr::attr_id, ib_mad_agent::class_version, ib_mad_hdr::class_version, ib_mad_transaction_operations::complete, DBGC, ENOTSUP, for_each_table_entry, ib_mad_agent::handle, hdr, ib_mad::hdr, ib_mad_tid::high, IB_MAD_AGENTS, IB_MGMT_CLASS_MASK, list_for_each_entry, ib_mad_tid::low, ib_mad_transaction::mad, ib_mad_interface::madx, memcmp(), ib_mad_agent::mgmt_class, ib_mad_hdr::mgmt_class, ntohl, ib_mad_transaction::op, and ib_mad_hdr::tid.

Referenced by ib_mi_complete_recv().

                                                         {
        struct ib_mad_hdr *hdr = &mad->hdr;
        struct ib_mad_transaction *madx;
        struct ib_mad_agent *agent;

        /* Look for a matching transaction by TID */
        list_for_each_entry ( madx, &mi->madx, list ) {
                if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
                              sizeof ( hdr->tid ) ) != 0 )
                        continue;
                /* Found a matching transaction */
                madx->op->complete ( ibdev, mi, madx, 0, mad, av );
                return 0;
        }

        /* If there is no matching transaction, look for a listening agent */
        for_each_table_entry ( agent, IB_MAD_AGENTS ) {
                if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
                       ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
                     ( agent->class_version != hdr->class_version ) ||
                     ( agent->attr_id != hdr->attr_id ) )
                        continue;
                /* Found a matching agent */
                agent->handle ( ibdev, mi, mad, av );
                return 0;
        }

        /* Otherwise, ignore it */
        DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
               mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
        return -ENOTSUP;
}
static void ib_mi_complete_recv ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct ib_address_vector *dest  __unused,
struct ib_address_vector source,
struct io_buffer iobuf,
int  rc 
) [static]

Complete receive via management interface.

Parameters:
ibdevInfiniband device
qpQueue pair
destDestination address vector
sourceSource address vector
iobufI/O buffer
rcCompletion status code

Definition at line 124 of file ib_mi.c.

References ib_mad_hdr::attr_id, ib_mad_hdr::base_version, ib_mad_hdr::class_version, io_buffer::data, DBGC, DBGC2_HDA, DBGC_HDA, free_iob(), hdr, ib_mad::hdr, ib_mad_tid::high, IB_MGMT_BASE_VERSION, ib_mi_handle(), ib_qp_get_ownerdata(), iob_len(), ib_mad_tid::low, mad, ib_mad_hdr::method, ib_mad_hdr::mgmt_class, ntohl, ntohs, out, ib_mad_hdr::status, strerror(), and ib_mad_hdr::tid.

                                                                    {
        struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
        union ib_mad *mad;
        struct ib_mad_hdr *hdr;

        /* Ignore errors */
        if ( rc != 0 ) {
                DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
                goto out;
        }

        /* Sanity checks */
        if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
                DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
                       mi, iob_len ( iobuf ) );
                DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
                goto out;
        }
        mad = iobuf->data;
        hdr = &mad->hdr;
        if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
                DBGC ( mi, "MI %p RX unsupported base version %x\n",
                       mi, hdr->base_version );
                DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
                goto out;
        }
        DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
               "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
               hdr->mgmt_class, hdr->class_version, hdr->method,
               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );

        /* Handle MAD */
        if ( ( rc = ib_mi_handle ( ibdev, mi, mad, source ) ) != 0 )
                goto out;

 out:
        free_iob ( iobuf );
}
int ib_mi_send ( struct ib_device ibdev,
struct ib_mad_interface mi,
union ib_mad mad,
struct ib_address_vector av 
)

Transmit MAD.

Parameters:
ibdevInfiniband device
miManagement interface
madMAD
avDestination address vector
Return values:
rcReturn status code

Definition at line 187 of file ib_mi.c.

References alloc_iob(), assert, ib_mad_hdr::attr_id, ib_mad_hdr::base_version, ib_mad_hdr::class_specific, ib_mad_hdr::class_version, DBGC, DBGC2_HDA, EINVAL, ENOMEM, free_iob(), hdr, ib_mad::hdr, ib_mad_tid::high, ib_smp_class_specific::hop_count, hop_count, ib_smp_class_specific::hop_pointer, hop_pointer, ib_smp_dr_path::hops, htonl, htons, IB_MGMT_BASE_VERSION, IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE, IB_MI_TID_MAGIC, ib_post_send(), IB_SMP_STATUS_D_INBOUND, iob_put, ib_mad_tid::low, ib_mad_smp::mad_hdr, memcpy(), ib_mad_hdr::method, ib_mad_hdr::mgmt_class, next_tid, ntohl, ntohs, ib_device::port, ib_mad_interface::qp, rc, ib_mad_smp::return_path, ib_mad_class_specific::smp, smp, ib_mad::smp, ib_mad_hdr::status, strerror(), and ib_mad_hdr::tid.

Referenced by ib_cm_send_drep(), ib_cm_send_rtu(), ib_mcast_leave(), ib_mi_timer_expired(), ib_sma_guid_info(), ib_sma_node_desc(), ib_sma_node_info(), ib_sma_pkey_table(), and ib_sma_port_info().

                                                                   {
        struct ib_mad_hdr *hdr = &mad->hdr;
        struct io_buffer *iobuf;
        int rc;

        /* Set common fields */
        hdr->base_version = IB_MGMT_BASE_VERSION;
        if ( ( hdr->tid.high == 0 ) && ( hdr->tid.low == 0 ) ) {
                hdr->tid.high = htonl ( IB_MI_TID_MAGIC );
                hdr->tid.low = htonl ( ++next_tid );
        }
        DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
               "%04x\n", mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
               hdr->mgmt_class, hdr->class_version, hdr->method,
               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );

        /* Construct directed route portion of response, if necessary */
        if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
                struct ib_mad_smp *smp = &mad->smp;
                unsigned int hop_pointer;
                unsigned int hop_count;

                smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
                hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
                hop_count = smp->mad_hdr.class_specific.smp.hop_count;
                assert ( hop_count == hop_pointer );
                if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
                                     sizeof ( smp->return_path.hops[0] ) ) ) {
                        smp->return_path.hops[hop_pointer] = ibdev->port;
                } else {
                        DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
                               "%d\n", mi, ntohl ( hdr->tid.high ),
                               ntohl ( hdr->tid.low ), hop_pointer );
                        return -EINVAL;
                }
        }

        /* Construct I/O buffer */
        iobuf = alloc_iob ( sizeof ( *mad ) );
        if ( ! iobuf ) {
                DBGC ( mi, "MI %p could not allocate buffer for TID "
                       "%08x%08x\n",
                       mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
                return -ENOMEM;
        }
        memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );

        /* Send I/O buffer */
        if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
                DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
                       mi,  ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ),
                       strerror ( rc ) );
                free_iob ( iobuf );
                return rc;
        }

        return 0;
}
static void ib_mi_timer_expired ( struct retry_timer timer,
int  expired 
) [static]

Handle management transaction timer expiry.

Parameters:
timerRetry timer
expiredFailure indicator

Definition at line 254 of file ib_mi.c.

References ib_mad_transaction::av, ib_mad_transaction_operations::complete, container_of, DBGC, ETIMEDOUT, hdr, ib_mad::hdr, ib_mad_tid::high, ib_mi_send(), ib_mad_interface::ibdev, ib_mad_tid::low, ib_mad_transaction::mad, ib_mad_transaction::mi, ntohl, NULL, ib_mad_transaction::op, start_timer(), and ib_mad_hdr::tid.

Referenced by ib_create_madx().

                                                                           {
        struct ib_mad_transaction *madx =
                container_of ( timer, struct ib_mad_transaction, timer );
        struct ib_mad_interface *mi = madx->mi;
        struct ib_device *ibdev = mi->ibdev;
        struct ib_mad_hdr *hdr = &madx->mad.hdr;

        /* Abandon transaction if we have tried too many times */
        if ( expired ) {
                DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
                       mi, ntohl ( hdr->tid.high ), ntohl ( hdr->tid.low ) );
                madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
                return;
        }

        /* Restart retransmission timer */
        start_timer ( timer );

        /* Resend MAD */
        ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
}
struct ib_mad_transaction* ib_create_madx ( struct ib_device ibdev,
struct ib_mad_interface mi,
union ib_mad mad,
struct ib_address_vector av,
struct ib_mad_transaction_operations op 
) [read]

Create management transaction.

Parameters:
ibdevInfiniband device
miManagement interface
madMAD to send
avDestination address, or NULL to use SM's GSI
opManagement transaction operations
Return values:
madxManagement transaction, or NULL

Definition at line 287 of file ib_mi.c.

References ib_mad_transaction::av, ib_mi_timer_expired(), IB_QKEY_GSI, IB_QPN_GSI, ib_address_vector::lid, ib_mad_transaction::list, list_add, ib_mad_transaction::mad, ib_mad_interface::madx, memcpy(), ib_mad_transaction::mi, NULL, ib_mad_transaction::op, op, ib_address_vector::qkey, ib_address_vector::qpn, ib_address_vector::sl, ib_device::sm_lid, ib_device::sm_sl, start_timer_nodelay(), ib_mad_transaction::timer, and zalloc().

Referenced by ib_cm_path_complete(), ib_create_path(), ib_create_service_madx(), ib_mcast_join(), and xsigo_xds_complete().

                                                            {
        struct ib_mad_transaction *madx;

        /* Allocate and initialise structure */
        madx = zalloc ( sizeof ( *madx ) );
        if ( ! madx )
                return NULL;
        timer_init ( &madx->timer, ib_mi_timer_expired, NULL );
        madx->mi = mi;
        madx->op = op;

        /* Determine address vector */
        if ( av ) {
                memcpy ( &madx->av, av, sizeof ( madx->av ) );
        } else {
                madx->av.lid = ibdev->sm_lid;
                madx->av.sl = ibdev->sm_sl;
                madx->av.qpn = IB_QPN_GSI;
                madx->av.qkey = IB_QKEY_GSI;
        }

        /* Copy MAD */
        memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );

        /* Add to list and start timer to send initial MAD */
        list_add ( &madx->list, &mi->madx );
        start_timer_nodelay ( &madx->timer );

        return madx;
}
void ib_destroy_madx ( struct ib_device *ibdev  __unused,
struct ib_mad_interface *mi  __unused,
struct ib_mad_transaction madx 
)

Destroy management transaction.

Parameters:
ibdevInfiniband device
miManagement interface
madxManagement transaction

Definition at line 327 of file ib_mi.c.

References free, ib_mad_transaction::list, list_del, stop_timer(), and ib_mad_transaction::timer.

Referenced by ib_cm_req_complete(), ib_create_path(), ib_destroy_conn(), ib_destroy_path(), ib_mcast_complete(), ib_mcast_join(), ib_mcast_leave(), ib_path_complete(), xsigo_discover(), xsigo_ib_notify(), xsigo_ib_remove(), xsigo_xcm_complete(), and xsigo_xds_complete().

                                                         {

        /* Stop timer and remove from list */
        stop_timer ( &madx->timer );
        list_del ( &madx->list );

        /* Free transaction */
        free ( madx );
}
int ib_create_mi ( struct ib_device ibdev,
enum ib_queue_pair_type  type,
struct ib_mad_interface **  new_mi 
)

Create management interface.

Parameters:
ibdevInfiniband device
typeQueue pair type
new_miNew management interface to fill in
Return values:
rcReturn status code

Definition at line 347 of file ib_mi.c.

References ib_mad_interface::cq, DBGC, ENOMEM, free, ib_create_cq(), ib_create_qp(), ib_destroy_cq(), ib_destroy_qp(), IB_MI_NUM_CQES, IB_MI_NUM_RECV_WQES, IB_MI_NUM_SEND_WQES, ib_modify_qp(), IB_QKEY_GSI, IB_QKEY_SMI, ib_qp_set_ownerdata(), IB_QPT_SMI, ib_refill_recv(), ib_mad_interface::ibdev, INIT_LIST_HEAD, ib_mad_interface::madx, ib_queue_pair::name, name, ib_queue_pair::qkey, ib_mad_interface::qp, ib_queue_pair::qpn, rc, strerror(), and zalloc().

Referenced by ib_open().

                                                      {
        struct ib_mad_interface *mi;
        const char *name;
        int rc;

        /* Allocate and initialise fields */
        mi = zalloc ( sizeof ( *mi ) );
        if ( ! mi ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        mi->ibdev = ibdev;
        INIT_LIST_HEAD ( &mi->madx );

        /* Create completion queue */
        if ( ( rc = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops,
                                   &mi->cq ) ) != 0 ) {
                DBGC ( mi, "MI %p could not create completion queue: %s\n",
                       mi, strerror ( rc ) );
                goto err_create_cq;
        }

        /* Create queue pair */
        name = ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" );
        if ( ( rc = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
                                   IB_MI_NUM_RECV_WQES, mi->cq,
                                   &ib_mi_queue_pair_ops, name, &mi->qp ) )!=0){
                DBGC ( mi, "MI %p could not create queue pair: %s\n",
                       mi, strerror ( rc ) );
                goto err_create_qp;
        }
        ib_qp_set_ownerdata ( mi->qp, mi );
        DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
               mi, mi->qp->name, mi->qp->qpn );

        /* Set queue key */
        mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
        if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
                DBGC ( mi, "MI %p could not set queue key: %s\n",
                       mi, strerror ( rc ) );
                goto err_modify_qp;
        }

        /* Fill receive ring */
        ib_refill_recv ( ibdev, mi->qp );
        *new_mi = mi;
        return 0;

 err_modify_qp:
        ib_destroy_qp ( ibdev, mi->qp );
 err_create_qp:
        ib_destroy_cq ( ibdev, mi->cq );
 err_create_cq:
        free ( mi );
 err_alloc:
        return rc;
}
void ib_destroy_mi ( struct ib_device ibdev,
struct ib_mad_interface mi 
)

Destroy management interface.

Parameters:
miManagement interface

Definition at line 411 of file ib_mi.c.

References ib_mad_transaction_operations::complete, ib_mad_interface::cq, DBGC, ECANCELED, free, ib_mad::hdr, ib_mad_tid::high, ib_destroy_cq(), ib_destroy_qp(), ib_mad_transaction::list, list_for_each_entry_safe, ib_mad_tid::low, ib_mad_transaction::mad, ib_mad_interface::madx, ntohl, NULL, ib_mad_transaction::op, ib_mad_interface::qp, and ib_mad_hdr::tid.

Referenced by ib_close(), and ib_open().

                                                                            {
        struct ib_mad_transaction *madx;
        struct ib_mad_transaction *tmp;

        /* Flush any outstanding requests */
        list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
                DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
                       mi, ntohl ( madx->mad.hdr.tid.high ),
                       ntohl ( madx->mad.hdr.tid.low ) );
                madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
        }

        ib_destroy_qp ( ibdev, mi->qp );
        ib_destroy_cq ( ibdev, mi->cq );
        free ( mi );
}

Variable Documentation

unsigned int next_tid [static]

TID to use for next MAD.

Definition at line 66 of file ib_mi.c.

Referenced by ib_mi_send().

Initial value:
 {
        .complete_recv = ib_mi_complete_recv,
}

Management interface completion operations.

Definition at line 169 of file ib_mi.c.

Initial value:
 {
        .alloc_iob = alloc_iob,
}

Management interface queue pair operations.

Definition at line 174 of file ib_mi.c.