62 { .
name =
"ib.post_send" };
66 { .
name =
"ib.post_recv" };
69 #define EINPROGRESS_INIT __einfo_error ( EINFO_EINPROGRESS_INIT ) 70 #define EINFO_EINPROGRESS_INIT __einfo_uniqify \ 71 ( EINFO_EINPROGRESS, 0x01, "Initialising" ) 72 #define EINPROGRESS_ARMED __einfo_error ( EINFO_EINPROGRESS_ARMED ) 73 #define EINFO_EINPROGRESS_ARMED __einfo_uniqify \ 74 ( EINFO_EINPROGRESS, 0x02, "Armed" ) 107 cq =
zalloc (
sizeof ( *cq ) );
120 DBGC (
ibdev,
"IBDEV %s could not initialise completion " 122 goto err_dev_create_cq;
125 DBGC (
ibdev,
"IBDEV %s created %d-entry completion queue %p (%p) " 147 DBGC (
ibdev,
"IBDEV %s destroying completion queue %#lx\n",
200 unsigned int num_send_wqes,
202 unsigned int num_recv_wqes,
213 total_size = (
sizeof ( *qp ) +
214 ( num_send_wqes *
sizeof (
qp->send.iobufs[0] ) ) +
215 ( num_recv_wqes *
sizeof (
qp->recv.iobufs[0] ) ) );
225 qp->send.is_send = 1;
226 qp->send.cq = send_cq;
228 qp->send.psn = (
random() & 0xffffffUL );
229 qp->send.num_wqes = num_send_wqes;
230 qp->send.iobufs = ( ( (
void * )
qp ) +
sizeof ( *qp ) );
232 qp->recv.cq = recv_cq;
234 qp->recv.psn = (
random() & 0xffffffUL );
235 qp->recv.num_wqes = num_recv_wqes;
236 qp->recv.iobufs = ( ( (
void * )
qp ) +
sizeof ( *qp ) +
237 ( num_send_wqes *
sizeof (
qp->send.iobufs[0] ) ));
244 DBGC (
ibdev,
"IBDEV %s could not initialise queue pair: " 246 goto err_dev_create_qp;
248 DBGC (
ibdev,
"IBDEV %s created queue pair %p (%p) with QPN %#lx\n",
250 DBGC (
ibdev,
"IBDEV %s QPN %#lx has %d send entries at [%p,%p)\n",
253 DBGC (
ibdev,
"IBDEV %s QPN %#lx has %d receive entries at [%p,%p)\n",
255 ( ( (
void * )
qp ) + total_size ) );
266 qp->ext_qpn =
qp->qpn;
269 if (
qp->ext_qpn !=
qp->qpn ) {
270 DBGC (
ibdev,
"IBDEV %s QPN %#lx has external QPN %#lx\n",
300 DBGC (
ibdev,
"IBDEV %s could not modify QPN %#lx: %s\n",
318 DBGC ( ibdev,
"IBDEV %s destroying QPN %#lx\n",
327 for ( i = 0 ; i <
qp->send.num_wqes ; i++ ) {
328 if ( ( iobuf =
qp->send.iobufs[i] ) !=
NULL )
331 for ( i = 0 ; i <
qp->recv.num_wqes ; i++ ) {
332 if ( ( iobuf =
qp->recv.iobufs[i] ) !=
NULL ) {
355 unsigned long qpn ) {
359 if ( (
qpn ==
qp->qpn ) || (
qpn ==
qp->ext_qpn ) )
380 sizeof (
mgid->gid ) ) == 0 ) {
426 if (
qp->send.fill >=
qp->send.num_wqes ) {
427 DBGC ( ibdev,
"IBDEV %s QPN %#lx send queue full\n",
437 memcpy ( &dest_copy,
dest,
sizeof ( dest_copy ) );
448 DBGC ( ibdev,
"IBDEV %s QPN %#lx could not post send WQE: " 479 DBGC ( ibdev,
"IBDEV %s QPN %#lx wrong RX buffer size (%zd)\n",
485 if (
qp->recv.fill >=
qp->recv.num_wqes ) {
486 DBGC ( ibdev,
"IBDEV %s QPN %#lx receive queue full\n",
493 DBGC ( ibdev,
"IBDEV %s QPN %#lx could not post receive WQE: " 518 if (
qp->send.cq->op->complete_send ) {
519 qp->send.cq->op->complete_send ( ibdev,
qp, iobuf,
rc );
541 if (
qp->recv.cq->op->complete_recv ) {
542 qp->recv.cq->op->complete_recv ( ibdev,
qp,
dest, source,
561 while (
qp->recv.fill <
qp->recv.num_wqes ) {
572 DBGC ( ibdev,
"IBDEV %s could not refill: %s\n",
616 default:
return "UNKNOWN";
639 DBGC ( ibdev,
"IBDEV %s link state is %s\n",
662 if ( (
rc = ibdev->
op->
open ( ibdev ) ) != 0 ) {
663 DBGC ( ibdev,
"IBDEV %s could not open: %s\n",
670 DBGC ( ibdev,
"IBDEV %s could not create SMI: %s\n",
677 DBGC ( ibdev,
"IBDEV %s could not create SMA: %s\n",
684 DBGC ( ibdev,
"IBDEV %s could not create GSI: %s\n",
771 goto err_dev_mcast_attach;
775 err_dev_mcast_attach:
827 DBGC ( ibdev,
"IBDEV %s does not support setting port " 828 "information\n", ibdev->
name );
833 DBGC ( ibdev,
"IBDEV %s could not set port information: %s\n",
852 DBGC ( ibdev,
"IBDEV %s does not support setting partition " 853 "key table\n", ibdev->
name );
858 DBGC ( ibdev,
"IBDEV %s could not set partition key table: " 922 total_len = (
sizeof ( *ibdev ) + priv_size );
923 ibdev =
zalloc ( total_len );
925 drv_priv = ( ( (
void * ) ibdev ) +
sizeof ( *ibdev ) );
949 if ( ibdev->
name[0] ==
'\0' ) {
958 DBGC ( ibdev,
"IBDEV %s registered (phys %s)\n", ibdev->
name,
963 if ( (
rc = driver->
probe ( ibdev ) ) != 0 ) {
964 DBGC ( ibdev,
"IBDEV %s could not add %s device: %s\n",
995 DBGC ( ibdev,
"IBDEV %s unregistered\n", ibdev->
name );
void unregister_ibdev(struct ib_device *ibdev)
Unregister Infiniband device.
int ib_set_pkey_table(struct ib_device *ibdev, union ib_mad *mad)
Set partition key table.
static void ib_notify(struct ib_device *ibdev)
Notify drivers of Infiniband device or link state change.
struct ib_mad_interface * smi
Subnet management interface.
void ib_destroy_sma(struct ib_device *ibdev __unused, struct ib_mad_interface *mi __unused)
Destroy subnet management agent and interface.
#define EINVAL
Invalid argument.
static __always_inline void ib_set_drvdata(struct ib_device *ibdev, void *priv)
Set Infiniband device driver-private data.
int ib_link_rc(struct ib_device *ibdev)
Get link state.
int(* set_port_info)(struct ib_device *ibdev, union ib_mad *mad)
Set port information.
void ib_poll_eq(struct ib_device *ibdev)
Poll event queue.
struct arbelprm_rc_send_wqe rc
Infiniband management interfaces.
void ib_destroy_mi(struct ib_device *ibdev, struct ib_mad_interface *mi)
Destroy management interface.
int(* set_pkey_table)(struct ib_device *ibdev, union ib_mad *mad)
Set partition key table.
#define IB_QPN_SMI
Subnet management interface QPN.
int(* probe)(struct ib_device *ibdev)
Probe device.
char name[IBDEV_NAME_LEN]
Name of this Infiniband device.
union ib_gid gid
Port GID (comprising GID prefix and port GUID)
#define EINFO_EINPROGRESS_INIT
struct ib_device * last_opened_ibdev(void)
Get most recently opened Infiniband device.
void(* poll_eq)(struct ib_device *ibdev)
Poll event queue.
#define list_add(new, head)
Add a new entry to the head of a list.
int(* open)(struct ib_device *ibdev)
Open port.
unsigned int open_count
Port open request counter.
int(* post_recv)(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf)
Post receive work queue entry.
void(* mcast_detach)(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Detach from multicast group.
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
uint32_t type
Operating system type.
#define __einfo_errortab(einfo)
#define IB_PORT_STATE_INIT
int ib_modify_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Modify queue pair.
int ib_create_cq(struct ib_device *ibdev, unsigned int num_cqes, struct ib_completion_queue_operations *op, struct ib_completion_queue **new_cq)
Create completion queue.
void ib_refill_recv(struct ib_device *ibdev, struct ib_queue_pair *qp)
Refill receive work queue.
int ib_open(struct ib_device *ibdev)
Open port.
struct device * dev
Underlying device.
int ib_set_port_info(struct ib_device *ibdev, union ib_mad *mad)
Set port information.
void(* notify)(struct ib_device *ibdev)
Notify of device or link state change.
int ib_mcast_attach(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Attach to multicast group.
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
struct ib_mad_interface * gsi
General services interface.
void * drv_priv
Driver private data.
static struct profiler ib_post_send_profiler __profiler
Post send work queue entry profiler.
An Infiniband upper-layer driver.
static const char * ib_link_state_text(struct ib_device *ibdev)
Textual representation of Infiniband link state.
#define IB_PKEY_DEFAULT
Default Infiniband partition key.
void ib_link_state_changed(struct ib_device *ibdev)
Notify of Infiniband link state change.
void ib_close(struct ib_device *ibdev)
Close port.
int ib_create_sma(struct ib_device *ibdev, struct ib_mad_interface *mi)
Create subnet management agent and interface.
#define for_each_table_entry_reverse(pointer, table)
Iterate through all entries within a linker table in reverse order.
A data structure for storing profiling information.
static void profile_stop(struct profiler *profiler)
Stop profiling.
static __always_inline struct ib_device * ibdev_get(struct ib_device *ibdev)
Get reference to Infiniband device.
An Infiniband Global Identifier.
struct list_head list
List of multicast GIDs on this QP.
Address Resolution Protocol constants and types.
#define IB_PORT_STATE_ARMED
#define ECANCELED
Operation canceled.
#define ENOTSUP
Operation not supported.
void ib_destroy_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Destroy completion queue.
A doubly-linked list entry (or list head)
unsigned int index
Index of this Infiniband device.
#define list_empty(list)
Test whether a list is empty.
int(* mcast_attach)(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Attach to multicast group.
#define list_first_entry(list, type, member)
Get the container of the first entry in a list.
struct ib_completion_queue * cq
Associated completion queue.
#define list_del(list)
Delete an entry from a list.
#define ENOMEM
Not enough space.
Infiniband completion queue operations.
void * memcpy(void *dest, const void *src, size_t len) __nonnull
Infiniband queue pair operations.
#define EINPROGRESS_ARMED
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
struct ib_device_operations * op
Infiniband operations.
An Infiniband Work Queue.
#define list_for_each_entry(pos, head, member)
Iterate over entries in a list.
#define __unused
Declare a variable or data structure as unused.
#define list_add_tail(new, head)
Add a new entry to the tail of a list.
void ib_complete_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf, int rc)
Complete send work queue entry.
REQUIRE_OBJECT(config_infiniband)
PERMANENT_PROCESS(ib_process, ib_step)
Infiniband event queue process.
unsigned long qpn
Queue pair number.
int(* modify_qp)(struct ib_device *ibdev, struct ib_queue_pair *qp)
Modify queue pair.
static struct list_head open_ib_devices
List of open Infiniband devices, in reverse order of opening.
#define IB_DRIVERS
Infiniband driver table.
static void profile_start(struct profiler *profiler)
Start profiling.
struct ib_work_queue * ib_find_wq(struct ib_completion_queue *cq, unsigned long qpn, int is_send)
Find work queue belonging to completion queue.
struct list_head list
List of queue pairs on this Infiniband device.
struct list_head work_queues
List of work queues completing to this queue.
#define ENOTCONN
The socket is not connected.
struct list_head list
List of Infiniband devices.
struct ib_completion_queue_operations * op
Completion queue operations.
#define IB_LID_NONE
Default Infiniband LID.
#define IB_QPN_GSI
General service interface QPN.
static __always_inline void ibdev_put(struct ib_device *ibdev)
Drop reference to Infiniband device.
char * strerror(int errno)
Retrieve string representation of error number.
static void(* free)(struct refcnt *refcnt))
ib_queue_pair_type
An Infiniband queue pair type.
void * zalloc(size_t size)
Allocate cleared memory.
struct ib_device * find_ibdev(union ib_gid *gid)
Find Infiniband device by GID.
struct ib_device * alloc_ibdev(size_t priv_size)
Allocate Infiniband device.
void ib_mcast_detach(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Detach from multicast group.
#define for_each_table_entry(pointer, table)
Iterate through all entries within a linker table.
long int random(void)
Generate a pseudo-random number between 0 and 2147483647L or 2147483562?
int ib_post_recv(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf)
Post receive work queue entry.
static size_t iob_tailroom(struct io_buffer *iobuf)
Calculate available space at end of an I/O buffer.
An Infiniband Completion Queue.
void ib_destroy_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Destroy queue pair.
int register_ibdev(struct ib_device *ibdev)
Register Infiniband device.
#define IB_PORT_STATE_DOWN
struct ib_queue_pair * qp
Containing queue pair.
struct list_head ib_devices
List of Infiniband devices.
int(* create_qp)(struct ib_device *ibdev, struct ib_queue_pair *qp)
Create queue pair.
struct ib_device * ibdev
Containing Infiniband device.
void(* destroy_cq)(struct ib_device *ibdev, struct ib_completion_queue *cq)
Destroy completion queue.
struct list_head list
List of work queues on this completion queue.
An Infiniband Queue Pair.
struct list_head cqs
List of completion queues.
Network device management.
struct arbelprm_qp_db_record qp
int ib_post_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf)
Post send work queue entry.
static uint16_t struct vmbus_xfer_pages_operations * op
#define INIT_LIST_HEAD(list)
Initialise a list head.
void(* destroy_qp)(struct ib_device *ibdev, struct ib_queue_pair *qp)
Destroy queue pair.
struct list_head list
List of completion queues on this Infiniband device.
static unsigned int ibdev_index
Infiniband device index.
#define ENOBUFS
No buffer space available.
struct ib_queue_pair * ib_find_qp_mgid(struct ib_device *ibdev, union ib_gid *gid)
Find queue pair by multicast GID.
static void ib_step(struct process *process __unused)
Single-step the Infiniband event queue.
int ib_create_qp(struct ib_device *ibdev, enum ib_queue_pair_type type, unsigned int num_send_wqes, struct ib_completion_queue *send_cq, unsigned int num_recv_wqes, struct ib_completion_queue *recv_cq, struct ib_queue_pair_operations *op, const char *name, struct ib_queue_pair **new_qp)
Create queue pair.
#define IB_PORT_STATE_ACTIVE
void(* remove)(struct ib_device *ibdev)
Remove device.
void ib_poll_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Poll completion queue.
uint8_t port_state
Port state.
void(* close)(struct ib_device *ibdev)
Close port.
unsigned long cqn
Completion queue number.
if(len >=6 *4) __asm__ __volatile__("movsl" if(len >=5 *4) __asm__ __volatile__("movsl" if(len >=4 *4) __asm__ __volatile__("movsl" if(len >=3 *4) __asm__ __volatile__("movsl" if(len >=2 *4) __asm__ __volatile__("movsl" if(len >=1 *4) __asm__ __volatile__("movsl" if((len % 4) >=2) __asm__ __volatile__("movsw" if((len % 2) >=1) __asm__ __volatile__("movsb" return dest
Infiniband subnet management agent.
unsigned int num_cqes
Number of completion queue entries.
int(* post_send)(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf)
Post send work queue entry.
struct errortab infiniband_errors [] __errortab
Human-readable message for the link statuses.
struct list_head qps
List of queue pairs.
REQUIRING_SYMBOL(register_ibdev)
void ib_complete_recv(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct ib_address_vector *source, struct io_buffer *iobuf, int rc)
Complete receive work queue entry.
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
struct list_head open_list
List of open Infiniband devices.
uint16_t pkey
Partition key.
An Infiniband Address Vector.
An Infiniband multicast GID.
#define LIST_HEAD_INIT(list)
Initialise a static list head.
int is_send
"Is a send queue" flag
#define for_each_table_entry_continue_reverse(pointer, table)
Iterate through all remaining entries within a linker table in reverse order.
struct ib_queue_pair * ib_find_qp_qpn(struct ib_device *ibdev, unsigned long qpn)
Find queue pair by QPN.
int memcmp(const void *first, const void *second, size_t len)
Compare memory regions.
static __always_inline void * ib_cq_get_drvdata(struct ib_completion_queue *cq)
Get Infiniband completion queue driver-private data.
struct ib_device * ibdev
Containing Infiniband device.
#define NULL
NULL pointer (VOID *)
void(* poll_cq)(struct ib_device *ibdev, struct ib_completion_queue *cq)
Poll completion queue.
int ib_create_mi(struct ib_device *ibdev, enum ib_queue_pair_type type, struct ib_mad_interface **new_mi)
Create management interface.
#define for_each_ibdev(ibdev)
Iterate over all network devices.
int(* create_cq)(struct ib_device *ibdev, struct ib_completion_queue *cq)
Create completion queue.
#define IB_MAX_PAYLOAD_SIZE
Maximum payload size.
#define EINFO_EINPROGRESS_ARMED